diff --git a/.gitignore b/.gitignore index 38bf9554b5b..59812ed6ed4 100644 --- a/.gitignore +++ b/.gitignore @@ -90,7 +90,6 @@ test.py litellm_config.yaml !.github/observatory/litellm_config.yaml .cursor -.vscode/launch.json litellm/proxy/to_delete_loadtest_work/* update_model_cost_map.py tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -100,4 +99,5 @@ STABILIZATION_TODO.md **/test-results **/playwright-report **/*.storageState.json -**/coverage \ No newline at end of file +**/coverage +test-config \ No newline at end of file diff --git a/.npmrc b/.npmrc index 168e81a1c4e..7999681cc35 100644 --- a/.npmrc +++ b/.npmrc @@ -2,4 +2,4 @@ # Packages needing lifecycle scripts: npm rebuild ignore-scripts=true # Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3d +min-release-age=3 diff --git a/Makefile b/Makefile index b6b674ff3b1..5dbd308a3e2 100644 --- a/Makefile +++ b/Makefile @@ -185,3 +185,6 @@ test-llm-translation-single: install-test-deps $(UV_RUN) pytest tests/llm_translation/$(FILE) \ --junitxml=test-results/junit.xml \ -v --tb=short --maxfail=100 --timeout=300 + +test-llm-translation-flush-vcr-cache: + $(UV_RUN) python tests/_flush_vcr_cache.py diff --git a/README.md b/README.md index d72fb746ed4..72fd43925c9 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Managing LLM calls across providers gets complicated fast — different SDKs, au Stripe image Google ADK - Greptile + Greptile OpenHands

Netflix

OpenAI Agents SDK diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index f6ed7767c46..4bfe9d31874 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -857,10 +857,16 @@ async def project_info( where={"team_id": project.team_id} ) if team: - is_team_member = ( - user_api_key_dict.user_id in team.admins - or user_api_key_dict.user_id in team.members - ) + caller_user_id = user_api_key_dict.user_id + for m in team.members_with_roles or []: + m_user_id = ( + m.get("user_id") + if isinstance(m, dict) + else getattr(m, "user_id", None) + ) + if m_user_id == caller_user_id: + is_team_member = True + break if not (is_admin or is_team_member): raise HTTPException( @@ -911,20 +917,20 @@ async def list_projects( include={"litellm_budget_table": True, "object_permission": True} ) else: - # Get projects for teams the user belongs to - user_teams = await prisma_client.db.litellm_teamtable.find_many( - where={ - "OR": [ - {"members": {"has": user_api_key_dict.user_id}}, - {"admins": {"has": user_api_key_dict.user_id}}, - ] - } + # Look up the user's team memberships via the reverse-index on + # LiteLLM_UserTable.teams (maintained by team_member_add alongside + # members_with_roles). This avoids a full scan of all team rows. + user_record = await prisma_client.db.litellm_usertable.find_unique( + where={"user_id": user_api_key_dict.user_id}, + ) + user_team_ids = ( + user_record.teams + if user_record is not None and user_record.teams + else [] ) - team_ids = [team.team_id for team in user_teams] - projects = await prisma_client.db.litellm_projecttable.find_many( - where={"team_id": {"in": team_ids}}, + where={"team_id": {"in": user_team_ids}}, include={"litellm_budget_table": True, "object_permission": True}, ) diff --git a/litellm-js/proxy/.npmrc b/litellm-js/proxy/.npmrc index 168e81a1c4e..7999681cc35 100644 --- a/litellm-js/proxy/.npmrc +++ b/litellm-js/proxy/.npmrc @@ -2,4 +2,4 @@ # Packages needing lifecycle scripts: npm rebuild ignore-scripts=true # Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3d +min-release-age=3 diff --git a/litellm-js/spend-logs/.npmrc b/litellm-js/spend-logs/.npmrc index 168e81a1c4e..7999681cc35 100644 --- a/litellm-js/spend-logs/.npmrc +++ b/litellm-js/spend-logs/.npmrc @@ -2,4 +2,4 @@ # Packages needing lifecycle scripts: npm rebuild ignore-scripts=true # Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3d +min-release-age=3 diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260429120000_search_tools_on_object_permission/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260429120000_search_tools_on_object_permission/migration.sql new file mode 100644 index 00000000000..bffdaaebc57 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260429120000_search_tools_on_object_permission/migration.sql @@ -0,0 +1,2 @@ +-- Search tool allowlists live on LiteLLM_ObjectPermissionTable (with agents, MCP, vector stores). +ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "search_tools" TEXT[] DEFAULT ARRAY[]::TEXT[]; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index fd54ed5243a..a9d3911c07b 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -277,6 +277,7 @@ model LiteLLM_ObjectPermissionTable { models String[] @default([]) blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user + search_tools String[] @default([]) // search_tool_name values this key/team/user may call teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] verification_tokens LiteLLM_VerificationToken[] diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 41c78296fbb..b8710da3438 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.69" +version = "0.4.70" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.69" +version = "0.4.70" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 23f444b1cea..259439d4d09 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -543,15 +543,17 @@ def _handle_retrieve_batch_providers_without_provider_config( ) else: raise litellm.exceptions.BadRequestError( - message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( - custom_llm_provider - ), + message=( + "LiteLLM doesn't support custom_llm_provider={} for 'retrieve_batch' without a `model` kwarg. " + "Supported via this path: 'openai', 'azure', 'vertex_ai', 'anthropic'. " + "'bedrock' is supported but requires `model` to be passed so the provider config can be loaded." + ).format(custom_llm_provider), model="n/a", llm_provider=custom_llm_provider, response=httpx.Response( status_code=400, content="Unsupported provider", - request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + request=httpx.Request(method="retrieve_batch", url="https://github.com/BerriAI/litellm"), # type: ignore ), ) return response diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index ce1bc26c5e0..11733ce4cee 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -432,9 +432,10 @@ class Cache: str: The final hashed cache key with the redis namespace. """ dynamic_cache_control: DynamicCacheControl = kwargs.get("cache", {}) + metadata = kwargs.get("metadata") or {} namespace = ( dynamic_cache_control.get("namespace") - or kwargs.get("metadata", {}).get("redis_namespace") + or metadata.get("redis_namespace") or self.namespace ) if namespace: diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 7d514e648fe..3cf1d911d7f 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -87,6 +87,18 @@ class CachingHandlerResponse(BaseModel): in_memory_cache_obj = InMemoryCache() +def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bool: + """ + When stream=True, do not run success callbacks at cache-hit time. + + Cached chat/text completion replay uses CustomStreamWrapper; cached Responses + replay uses CachedResponsesAPIStreamingIterator. Both invoke logging success + handlers when the stream finishes; firing them here too would double-count + spend and callback records. + """ + return kwargs.get("stream", False) is True + + class LLMCachingHandler: def __init__( self, @@ -99,6 +111,7 @@ class LLMCachingHandler: self.async_streaming_chunks: List[ModelResponse] = [] self.sync_streaming_chunks: List[ModelResponse] = [] self.request_kwargs = request_kwargs + self.preset_cache_key: Optional[str] = None self.original_function = original_function self.start_time = start_time if litellm.cache is not None and isinstance(litellm.cache.cache, RedisCache): @@ -206,7 +219,7 @@ class LLMCachingHandler: custom_llm_provider=kwargs.get("custom_llm_provider", None), args=args, ) - if kwargs.get("stream", False) is False: + if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs): # LOG SUCCESS self._async_log_cache_hit_on_callbacks( logging_obj=logging_obj, @@ -215,11 +228,12 @@ class LLMCachingHandler: end_time=end_time, cache_hit=cache_hit, ) - cache_key = litellm.cache.get_cache_key(**kwargs) - if ( - isinstance(cached_result, BaseModel) - or isinstance(cached_result, CustomStreamWrapper) - ) and hasattr(cached_result, "_hidden_params"): + cache_key = ( + self.preset_cache_key + or self.request_kwargs.get("cache_key") + or litellm.cache.get_cache_key(**self.request_kwargs) + ) + if hasattr(cached_result, "_hidden_params"): cached_result._hidden_params["cache_key"] = cache_key # type: ignore return CachingHandlerResponse(cached_result=cached_result) elif ( @@ -265,8 +279,6 @@ class LLMCachingHandler: kwargs: Dict[str, Any], args: Optional[Tuple[Any, ...]] = None, ) -> CachingHandlerResponse: - from litellm.utils import CustomStreamWrapper - cached_result: Optional[Any] = None # Check if caching should be performed BEFORE doing expensive kwargs copy @@ -282,6 +294,11 @@ class LLMCachingHandler: args, ) ) + if new_kwargs.get("metadata") is None: + new_kwargs.pop("metadata", None) + if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs: + new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs) + self.request_kwargs = new_kwargs print_verbose("Checking Sync Cache") cached_result = litellm.cache.get_cache(**new_kwargs) if cached_result is not None: @@ -322,17 +339,19 @@ class LLMCachingHandler: is_async=False, ) - logging_obj.handle_sync_success_callbacks_for_async_calls( - result=cached_result, - start_time=start_time, - end_time=end_time, - cache_hit=cache_hit, + if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs): + logging_obj.handle_sync_success_callbacks_for_async_calls( + result=cached_result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + ) + cache_key = ( + self.preset_cache_key + or self.request_kwargs.get("cache_key") + or litellm.cache.get_cache_key(**self.request_kwargs) ) - cache_key = litellm.cache.get_cache_key(**kwargs) - if ( - isinstance(cached_result, BaseModel) - or isinstance(cached_result, CustomStreamWrapper) - ) and hasattr(cached_result, "_hidden_params"): + if hasattr(cached_result, "_hidden_params"): cached_result._hidden_params["cache_key"] = cache_key # type: ignore return CachingHandlerResponse(cached_result=cached_result) return CachingHandlerResponse(cached_result=cached_result) @@ -686,6 +705,11 @@ class LLMCachingHandler: args, ) ) + if new_kwargs.get("metadata") is None: + new_kwargs.pop("metadata", None) + if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs: + new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs) + self.request_kwargs = new_kwargs cached_result: Optional[Any] = None if call_type == CallTypes.aembedding.value: if isinstance(new_kwargs["input"], str): @@ -710,14 +734,26 @@ class LLMCachingHandler: if all(result is None for result in cached_result): cached_result = None else: + request_kwargs = new_kwargs.copy() + request_cache_key = request_kwargs.pop("cache_key", None) if litellm.cache._supports_async() is True: ## check if dual cache is supported ## + self.preset_cache_key = ( + request_cache_key or litellm.cache.get_cache_key(**request_kwargs) + ) cached_result = await litellm.cache.async_get_cache( - dynamic_cache_object=self.dual_cache, **new_kwargs + dynamic_cache_object=self.dual_cache, + cache_key=self.preset_cache_key, + **request_kwargs, ) else: # fallback for caches that don't support async + self.preset_cache_key = ( + request_cache_key or litellm.cache.get_cache_key(**request_kwargs) + ) cached_result = litellm.cache.get_cache( - dynamic_cache_object=self.dual_cache, **new_kwargs + dynamic_cache_object=self.dual_cache, + cache_key=self.preset_cache_key, + **request_kwargs, ) return cached_result @@ -825,8 +861,27 @@ class LLMCachingHandler: elif (call_type == "aresponses" or call_type == "responses") and isinstance( cached_result, dict ): - # Convert cached dict back to ResponsesAPIResponse object - cached_result = ResponsesAPIResponse(**cached_result) + from litellm.responses.streaming_iterator import ( + CachedResponsesAPIStreamingIterator, + ) + + response_obj = ResponsesAPIResponse(**cached_result) + if ( + hasattr(response_obj, "_hidden_params") + and response_obj._hidden_params is not None + and isinstance(response_obj._hidden_params, dict) + ): + response_obj._hidden_params["cache_hit"] = True + + if kwargs.get("stream", False) is True: + cached_result = CachedResponsesAPIStreamingIterator( + response=response_obj, + logging_obj=logging_obj, + request_data=kwargs, + call_type=call_type, + ) + else: + cached_result = response_obj if ( hasattr(cached_result, "_hidden_params") diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 34ae3638a5b..8060a65b78d 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -92,6 +92,25 @@ class DualCache(BaseCache): if default_redis_ttl is not None: self.default_redis_ttl = default_redis_ttl + def attach_redis_cache( + self, + redis_cache: Optional[RedisCache] = None, + *, + default_redis_ttl: Optional[float] = None, + ) -> None: + """ + Attach a Redis backend if this DualCache does not already have one. + + No-op when ``redis_cache`` is None or when Redis was already set (constructor + or a prior attach). Use this for lazy wiring after a shared Redis client exists. + Does not backfill in-memory-only keys to Redis. + """ + if redis_cache is None or self.redis_cache is not None: + return + self.redis_cache = redis_cache + if default_redis_ttl is not None: + self.default_redis_ttl = default_redis_ttl + def set_cache(self, key, value, local_only: bool = False, **kwargs): # Update both Redis and in-memory cache try: @@ -392,6 +411,7 @@ class DualCache(BaseCache): value: float, parent_otel_span: Optional[Span] = None, local_only: bool = False, + refresh_ttl: bool = False, **kwargs, ) -> Optional[float]: """ @@ -399,6 +419,9 @@ class DualCache(BaseCache): Value - float - the value you want to increment by + Refresh_ttl - bool - if True, resets the Redis TTL on every write. + Default False preserves window-style semantics. + Returns - the incremented value, or None if no cache backend is available (in_memory_cache is None and Redis failed/is absent). """ @@ -415,6 +438,7 @@ class DualCache(BaseCache): value, parent_otel_span=parent_otel_span, ttl=kwargs.get("ttl", None), + refresh_ttl=refresh_ttl, ) return result diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 84a2887f527..cb9ce475d30 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -551,6 +551,13 @@ class RedisCache(BaseCache): async def async_set_cache(self, key, value, **kwargs): from redis.asyncio import Redis + if key is None: + verbose_logger.debug( + "LiteLLM Redis Caching: async set() skipped — key is None, value=%r", + value, + ) + return None + start_time = time.time() try: _redis_client: Redis = self.init_async_client() # type: ignore @@ -569,8 +576,9 @@ class RedisCache(BaseCache): ) ) verbose_logger.error( - "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s", + "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, key=%r, value=%r", str(e), + key, value, ) raise e @@ -824,6 +832,7 @@ class RedisCache(BaseCache): value: float, ttl: Optional[int] = None, parent_otel_span: Optional[Span] = None, + refresh_ttl: bool = False, ) -> float: from redis.asyncio import Redis @@ -834,11 +843,12 @@ class RedisCache(BaseCache): try: result = await _redis_client.incrbyfloat(name=key, amount=value) if _used_ttl is not None: - # check if key already has ttl, if not -> set ttl - current_ttl = await _redis_client.ttl(key) - if current_ttl == -1: - # Key has no expiration + if refresh_ttl: await _redis_client.expire(key, _used_ttl) + else: + current_ttl = await _redis_client.ttl(key) + if current_ttl == -1: + await _redis_client.expire(key, _used_ttl) ## LOGGING ## end_time = time.time() diff --git a/litellm/constants.py b/litellm/constants.py index a0e99dd16b7..6c889a317b8 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1393,6 +1393,10 @@ except (ValueError, TypeError): LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check" LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli" LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs" +# Stable identifier substituted in place of the master key on UserAPIKeyAuth +# objects so the master key (or its hash) never propagates to spend logs, +# Prometheus metrics, audit trails, or any other downstream consumer. +LITELLM_PROXY_MASTER_KEY_ALIAS = "litellm_proxy_master_key" # Key Rotation Constants LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false") @@ -1421,6 +1425,7 @@ LITELLM_PROXY_ADMIN_NAME = "default_user_id" LITELLM_CLI_SOURCE_IDENTIFIER = "litellm-cli" LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token" CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session" +CLI_SSO_SESSION_TTL_SECONDS = 600 CLI_JWT_TOKEN_NAME = "cli-jwt-token" # Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility CLI_JWT_EXPIRATION_HOURS = int( diff --git a/litellm/integrations/arize/arize_phoenix_client.py b/litellm/integrations/arize/arize_phoenix_client.py index 3c83517bb55..8c3c2a5ff0f 100644 --- a/litellm/integrations/arize/arize_phoenix_client.py +++ b/litellm/integrations/arize/arize_phoenix_client.py @@ -2,11 +2,23 @@ Arize Phoenix API client for fetching prompt versions from Arize Phoenix. """ +import urllib.parse from typing import Any, Dict, Optional from litellm.llms.custom_httpx.http_handler import HTTPHandler +def _sanitize_id(identifier: str) -> str: + """Reject path traversal characters and URL-encode the identifier.""" + if any(c in identifier for c in ("/", "\\", "#", "?")): + raise ValueError( + f"Invalid identifier {identifier!r}: contains disallowed characters" + ) + if ".." in identifier: + raise ValueError(f"Invalid identifier {identifier!r}: path traversal detected") + return urllib.parse.quote(identifier, safe="") + + class ArizePhoenixClient: """ Client for interacting with Arize Phoenix API to fetch prompt versions. @@ -53,7 +65,8 @@ class ArizePhoenixClient: Returns: Dictionary containing prompt version data, or None if not found """ - url = f"{self.api_base}/v1/prompt_versions/{prompt_version_id}" + safe_id = _sanitize_id(prompt_version_id) + url = f"{self.api_base}/v1/prompt_versions/{safe_id}" try: # Use the underlying httpx client directly to avoid query param extraction diff --git a/litellm/integrations/bitbucket/bitbucket_client.py b/litellm/integrations/bitbucket/bitbucket_client.py index 0502422cf8b..e742cc14b7d 100644 --- a/litellm/integrations/bitbucket/bitbucket_client.py +++ b/litellm/integrations/bitbucket/bitbucket_client.py @@ -3,11 +3,27 @@ BitBucket API client for fetching .prompt files from BitBucket repositories. """ import base64 +import urllib.parse from typing import Any, Dict, List, Optional from litellm.llms.custom_httpx.http_handler import HTTPHandler +def _sanitize_file_path(file_path: str) -> str: + """Reject path traversal and URL-encode each path segment.""" + if "#" in file_path or "?" in file_path: + raise ValueError( + f"Invalid file path {file_path!r}: contains URL special characters" + ) + parts = file_path.split("/") + for part in parts: + if part == "..": + raise ValueError( + f"Invalid file path {file_path!r}: path traversal detected" + ) + return "/".join(urllib.parse.quote(part, safe="") for part in parts) + + class BitBucketClient: """ Client for interacting with BitBucket API to fetch .prompt files. @@ -72,7 +88,8 @@ class BitBucketClient: Returns: File content as string, or None if file not found """ - url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}" + safe_path = _sanitize_file_path(file_path) + url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}" try: response = self.http_handler.get(url, headers=self.headers) @@ -119,7 +136,8 @@ class BitBucketClient: Returns: List of file paths """ - url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{directory_path}" + safe_dir = _sanitize_file_path(directory_path) if directory_path else "" + url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_dir}" try: response = self.http_handler.get(url, headers=self.headers) @@ -211,7 +229,8 @@ class BitBucketClient: Returns: Dictionary containing file metadata, or None if file not found """ - url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}" + safe_path = _sanitize_file_path(file_path) + url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}" try: # Use GET with Range header to get just the headers (HEAD equivalent) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 723b142dfad..d9e57ee7cee 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -265,6 +265,7 @@ class PrometheusLogger(CustomLogger): ######################################## # LiteLLM Virtual API KEY metrics ######################################## + # Remaining MODEL RPM limit for API Key self.litellm_remaining_api_key_requests_for_model = self._gauge_factory( "litellm_remaining_api_key_requests_for_model", diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index e2e304931a4..3776d276912 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -31,15 +31,23 @@ def load_cli_token() -> Optional[dict]: return None -def get_litellm_gateway_api_key() -> Optional[str]: +def get_litellm_gateway_api_key( + expected_base_url: Optional[str] = None, +) -> Optional[str]: """ Get the stored CLI API key for use with LiteLLM SDK. This function reads the token file created by `litellm-proxy login` and returns the API key for use in Python scripts. + Args: + expected_base_url: When provided, the key is only returned if it was + originally issued for this URL. Pass the target server URL to + prevent credential leakage when the client is pointed at a + different (possibly malicious) server. + Returns: - str: The API key if found, None otherwise + str: The API key if found (and origin matches), None otherwise Example: >>> import litellm @@ -53,6 +61,10 @@ def get_litellm_gateway_api_key() -> Optional[str]: >>> ) """ token_data = load_cli_token() - if token_data and "key" in token_data: - return token_data["key"] - return None + if not token_data or "key" not in token_data: + return None + if expected_base_url is not None: + stored_url = token_data.get("base_url") + if stored_url != expected_base_url.rstrip("/"): + return None + return token_data["key"] diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 563609af1e4..ffb6436f387 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -23,6 +23,13 @@ def _raise_env_reference_error(param: str, *, source: str) -> None: ) +def validate_no_callback_env_reference( + param: str, value: object, *, source: str +) -> None: + if _is_env_reference(value): + _raise_env_reference_error(param, source=source) + + # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict _supported_callback_params = [ "langfuse_public_key", @@ -66,8 +73,9 @@ def initialize_standard_callback_dynamic_params( for param in _supported_callback_params: if param in kwargs: _param_value = kwargs.get(param) - if _is_env_reference(_param_value): - _raise_env_reference_error(param, source="request body") + validate_no_callback_env_reference( + param, _param_value, source="request body" + ) standard_callback_dynamic_params[param] = _param_value # type: ignore # 2. Fallback: check "metadata" or "litellm_params" -> "metadata" @@ -80,8 +88,9 @@ def initialize_standard_callback_dynamic_params( for param in _supported_callback_params: if param not in standard_callback_dynamic_params and param in metadata: _param_value = metadata.get(param) - if _is_env_reference(_param_value): - _raise_env_reference_error(param, source="metadata") + validate_no_callback_env_reference( + param, _param_value, source="metadata" + ) standard_callback_dynamic_params[param] = _param_value # type: ignore return standard_callback_dynamic_params diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index f5f28822ca1..7be70852978 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -77,8 +77,8 @@ def get_proxy_server_request_headers(litellm_params: Optional[dict]) -> dict: if litellm_params is None: return {} - proxy_request_headers = ( - litellm_params.get("proxy_server_request", {}).get("headers", {}) or {} - ) + proxy_request_headers = (litellm_params.get("proxy_server_request") or {}).get( + "headers" + ) or {} return proxy_request_headers diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fe8387476ee..ba840bc3d89 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1661,6 +1661,20 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: return sanitized +_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES = {"application/pdf", "text/plain"} + + +def _is_anthropic_document_data_uri(url: str) -> bool: + # Anthropic's base64 document source accepts only application/pdf and + # text/plain (see select_anthropic_content_block_type_for_file). Routing + # other mimes here would produce a document block the API rejects, so we + # leave them on the image code path. + match = re.match(r"data:([^;,]+)", url) + if not match: + return False + return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES + + def convert_to_anthropic_tool_result( message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage], force_base64: bool = False, @@ -1698,14 +1712,24 @@ def convert_to_anthropic_tool_result( """ anthropic_content: Union[ str, - List[Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]], + List[ + Union[ + AnthropicMessagesToolResultContent, + AnthropicMessagesImageParam, + AnthropicMessagesDocumentParam, + ] + ], ] = "" if isinstance(message["content"], str): anthropic_content = message["content"] elif isinstance(message["content"], List): content_list = message["content"] anthropic_content_list: List[ - Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam] + Union[ + AnthropicMessagesToolResultContent, + AnthropicMessagesImageParam, + AnthropicMessagesDocumentParam, + ] ] = [] for content in content_list: if content["type"] == "text": @@ -1720,21 +1744,62 @@ def convert_to_anthropic_tool_result( text_content["cache_control"] = cache_control_value anthropic_content_list.append(text_content) elif content["type"] == "image_url": + image_url_value = content["image_url"] format = ( - content["image_url"].get("format") - if isinstance(content["image_url"], dict) + image_url_value.get("format") + if isinstance(image_url_value, dict) else None ) - _anthropic_image_param = create_anthropic_image_param( - content["image_url"], format=format, is_bedrock_invoke=force_base64 + url_str = ( + image_url_value.get("url") + if isinstance(image_url_value, dict) + else image_url_value ) - _anthropic_image_param = add_cache_control_to_content( - anthropic_content_element=_anthropic_image_param, + # Data URIs with non-image mime types (e.g. application/pdf) must + # translate to Anthropic document blocks, not image blocks — + # wrapping a PDF in `type: "image"` is rejected by the API. + if isinstance(url_str, str) and _is_anthropic_document_data_uri( + url_str + ): + synth_file_message: ChatCompletionFileObject = { + "type": "file", + "file": {"file_data": url_str}, + } + _document_block = anthropic_process_openai_file_message( + synth_file_message + ) + _document_block = add_cache_control_to_content( + anthropic_content_element=cast( + AnthropicMessagesDocumentParam, _document_block + ), + original_content_element=content, + ) + anthropic_content_list.append( + cast(AnthropicMessagesDocumentParam, _document_block) + ) + else: + _anthropic_image_param = create_anthropic_image_param( + image_url_value, + format=format, + is_bedrock_invoke=force_base64, + ) + _anthropic_image_param = add_cache_control_to_content( + anthropic_content_element=_anthropic_image_param, + original_content_element=content, + ) + anthropic_content_list.append( + cast(AnthropicMessagesImageParam, _anthropic_image_param) + ) + elif content["type"] == "file": + file_content = cast(ChatCompletionFileObject, content) + _file_block = anthropic_process_openai_file_message(file_content) + _file_block = add_cache_control_to_content( + anthropic_content_element=cast( + AnthropicMessagesDocumentParam, _file_block + ), original_content_element=content, ) - anthropic_content_list.append( - cast(AnthropicMessagesImageParam, _anthropic_image_param) - ) + anthropic_content_list.append(_file_block) anthropic_content = anthropic_content_list anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None @@ -3977,6 +4042,55 @@ def _convert_to_bedrock_tool_call_result( tool_result_content_blocks.append( BedrockToolResultContentBlock(image=_block["image"]) ) + elif "document" in _block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(document=_block["document"]) + ) + else: + verbose_logger.warning( + "Bedrock Converse: unrecognized BedrockContentBlock keys " + "%s for image_url tool-result block %s; dropping.", + list(_block.keys()), + content, + ) + elif content["type"] == "file": + # Match the user-message path (_process_file_message): accept + # either file_data (base64 data URI) or file_id (server-side + # reference / URL) and hand off to BedrockImageProcessor. Raise + # BadRequestError on both-None rather than silently dropping. + file_obj = content.get("file") or {} + file_data = file_obj.get("file_data") + file_id = file_obj.get("file_id") + if file_data is None and file_id is None: + raise litellm.BadRequestError( + message="file_data and file_id cannot both be None. Got={}".format( + content + ), + model="", + llm_provider="bedrock", + ) + file_format = file_obj.get("format") + _file_block: BedrockContentBlock = ( + BedrockImageProcessor.process_image_sync( + image_url=cast(str, file_id or file_data), + format=file_format, + ) + ) + if "document" in _file_block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(document=_file_block["document"]) + ) + elif "image" in _file_block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(image=_file_block["image"]) + ) + else: + verbose_logger.warning( + "Bedrock Converse: unrecognized BedrockContentBlock keys " + "%s for file tool-result block %s; dropping.", + list(_file_block.keys()), + content, + ) message.get("name", "") id = str(message.get("tool_call_id", str(uuid.uuid4()))) @@ -4468,6 +4582,11 @@ class BedrockConverseMessagesProcessor: message=cast(ChatCompletionFileObject, element) ) _parts.append(_part) + elif element["type"] == "document": + _part = BedrockConverseMessagesProcessor._process_document_message( + element + ) + _parts.append(_part) _cache_point_block = ( litellm.AmazonConverseConfig()._get_cache_point_block( message_block=cast( @@ -4750,6 +4869,44 @@ class BedrockConverseMessagesProcessor: image_url=cast(str, file_id or file_data), format=format ) + @staticmethod + def _process_document_message(element: dict) -> BedrockContentBlock: + """Convert a document content block to a Bedrock DocumentBlock. + + Handles the Anthropic-style document format: + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "..."}} + """ + source = element["source"] + source_type = source.get("type") + if source_type != "base64": + raise ValueError( + f"Bedrock Converse only supports base64-encoded document sources, got '{source_type}'. " + "Please convert the document to base64 before sending to Bedrock." + ) + media_type: str = source["media_type"] + data: str = source["data"] + doc_format = BedrockImageProcessor._validate_format( + mime_type=media_type, image_format=media_type.split("/")[1] + ) + + # Deterministic name using the same hashing pattern as _create_bedrock_block + HASH_SAMPLE_BYTES = 64 * 1024 + normalized = "".join(data.split()).encode("utf-8") + sample = normalized[:HASH_SAMPLE_BYTES] + hasher = hashlib.sha256() + hasher.update(sample) + hasher.update(str(len(normalized)).encode("utf-8")) + content_hash = hasher.hexdigest()[:16] + document_name = f"Document_{content_hash}_{doc_format}" + + return BedrockContentBlock( + document=BedrockDocumentBlock( + source=BedrockSourceBlock(bytes=data), + format=doc_format, + name=document_name, + ) + ) + @staticmethod def add_thinking_blocks_to_assistant_content( thinking_blocks: List[BedrockContentBlock], @@ -4847,6 +5004,11 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 ) ) _parts.append(_part) + elif element["type"] == "document": + _part = BedrockConverseMessagesProcessor._process_document_message( + element + ) + _parts.append(_part) _cache_point_block = ( litellm.AmazonConverseConfig()._get_cache_point_block( message_block=cast( diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index f3f560b33b9..dbc9cabdc7a 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -60,6 +60,9 @@ def _redact_choice_content(choice): def _redact_responses_api_output(output_items): """Helper to redact ResponsesAPIResponse output items.""" for output_item in output_items: + if hasattr(output_item, "text"): + output_item.text = "redacted-by-litellm" + if hasattr(output_item, "content") and isinstance(output_item.content, list): for content_part in output_item.content: if hasattr(content_part, "text"): @@ -75,6 +78,28 @@ def _redact_responses_api_output(output_items): summary_item.text = "redacted-by-litellm" +def _redact_responses_api_output_dict(output_items, redacted_str: str): + """Helper to redact ResponsesAPIResponse output items in dict form.""" + for output_item in output_items: + if not isinstance(output_item, dict): + continue + + if "text" in output_item: + output_item["text"] = redacted_str + + if isinstance(output_item.get("content"), list): + for content_item in output_item["content"]: + if isinstance(content_item, dict) and "text" in content_item: + content_item["text"] = redacted_str + + if output_item.get("type") == "reasoning" and isinstance( + output_item.get("summary"), list + ): + for summary_item in output_item["summary"]: + if isinstance(summary_item, dict) and "text" in summary_item: + summary_item["text"] = redacted_str + + def _redact_standard_logging_object(model_call_details: dict): """Redact messages and response inside standard_logging_object if present.""" standard_logging_object = model_call_details.get("standard_logging_object") @@ -93,28 +118,11 @@ def _redact_standard_logging_object(model_call_details: dict): if isinstance(response, dict) and "output" in response: # ResponsesAPIResponse format - redact content in output items if isinstance(response.get("output"), list): - for output_item in response["output"]: - if isinstance(output_item, dict) and "content" in output_item: - if isinstance(output_item["content"], list): - for content_item in output_item["content"]: - if ( - isinstance(content_item, dict) - and "text" in content_item - ): - content_item["text"] = redacted_str + _redact_responses_api_output_dict(response["output"], redacted_str) elif isinstance(response, dict) and "choices" in response: # ModelResponse dict format - redact content in choices if isinstance(response.get("choices"), list): - for choice in response["choices"]: - if isinstance(choice, dict): - if "message" in choice and isinstance(choice["message"], dict): - choice["message"]["content"] = redacted_str - if "audio" in choice["message"]: - choice["message"]["audio"] = None - elif "delta" in choice and isinstance(choice["delta"], dict): - choice["delta"]["content"] = redacted_str - if "audio" in choice["delta"]: - choice["delta"]["audio"] = None + _redact_model_response_dict_choices(response["choices"], redacted_str) elif isinstance(response, str): standard_logging_object["response"] = redacted_str else: @@ -122,6 +130,29 @@ def _redact_standard_logging_object(model_call_details: dict): standard_logging_object["response"] = {"text": redacted_str} +def _redact_model_response_dict_choices(choices, redacted_str: str): + for choice in choices: + if isinstance(choice, dict): + if "message" in choice and isinstance(choice["message"], dict): + choice["message"]["content"] = redacted_str + if "reasoning_content" in choice["message"]: + choice["message"]["reasoning_content"] = redacted_str + if "thinking_blocks" in choice["message"]: + choice["message"]["thinking_blocks"] = None + if "audio" in choice["message"]: + choice["message"]["audio"] = None + elif "delta" in choice and isinstance(choice["delta"], dict): + choice["delta"]["content"] = redacted_str + if "reasoning_content" in choice["delta"]: + choice["delta"]["reasoning_content"] = redacted_str + if "thinking_blocks" in choice["delta"]: + choice["delta"]["thinking_blocks"] = None + if "audio" in choice["delta"]: + choice["delta"]["audio"] = None + else: + _redact_choice_content(choice) + + def perform_redaction(model_call_details: dict, result): """ Performs the actual redaction on the logging object and result. @@ -132,6 +163,7 @@ def perform_redaction(model_call_details: dict, result): ] model_call_details["prompt"] = "" model_call_details["input"] = "" + _redact_standard_logging_object(model_call_details) # Redact streaming response if ( @@ -171,30 +203,14 @@ def perform_redaction(model_call_details: dict, result): elif isinstance(_result, dict) and "choices" in _result: # Handle dict representation of ModelResponse (e.g., from model_dump()) if _result.get("choices") is not None: - for choice in _result["choices"]: - if isinstance(choice, dict): - if "message" in choice and isinstance(choice["message"], dict): - choice["message"]["content"] = "redacted-by-litellm" - if "reasoning_content" in choice["message"]: - choice["message"][ - "reasoning_content" - ] = "redacted-by-litellm" - if "thinking_blocks" in choice["message"]: - choice["message"]["thinking_blocks"] = None - if "audio" in choice["message"]: - choice["message"]["audio"] = None - elif "delta" in choice and isinstance(choice["delta"], dict): - choice["delta"]["content"] = "redacted-by-litellm" - if "reasoning_content" in choice["delta"]: - choice["delta"][ - "reasoning_content" - ] = "redacted-by-litellm" - if "thinking_blocks" in choice["delta"]: - choice["delta"]["thinking_blocks"] = None - if "audio" in choice["delta"]: - choice["delta"]["audio"] = None - else: - _redact_choice_content(choice) + _redact_model_response_dict_choices( + _result["choices"], "redacted-by-litellm" + ) + elif isinstance(_result, dict) and "output" in _result: + if isinstance(_result.get("output"), list): + _redact_responses_api_output_dict( + _result["output"], "redacted-by-litellm" + ) elif isinstance(_result, litellm.ResponsesAPIResponse): if hasattr(_result, "output"): _redact_responses_api_output(_result.output) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 8b88ef94821..d7803455b4a 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -21,6 +21,8 @@ class SensitiveDataMasker: "auth", "authorization", "credential", + # Plural form: Vertex uses ``vertex_credentials``; segment-exact + # matching otherwise misses it because "credential" != "credentials". "credentials", "access", "private", diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index e281b172685..fa7faf3035d 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2244,7 +2244,7 @@ class CustomStreamWrapper: asyncio.create_task( self.logging_obj.async_failure_handler(e, traceback_exception) ) - raise e + self._handle_stream_fallback_error(e) except Exception as e: traceback_exception = traceback.format_exc() if self.logging_obj is not None: diff --git a/litellm/litellm_core_utils/url_utils.py b/litellm/litellm_core_utils/url_utils.py index a65d0892aa2..224927e5acd 100644 --- a/litellm/litellm_core_utils/url_utils.py +++ b/litellm/litellm_core_utils/url_utils.py @@ -22,7 +22,7 @@ Admins can opt out via two ``litellm`` globals (wired from proxy config): import socket from ipaddress import ip_address, ip_network from typing import Any, List, Set, Tuple -from urllib.parse import urlparse, urlunparse +from urllib.parse import quote, urlparse, urlunparse import httpx @@ -46,6 +46,46 @@ class SSRFError(ValueError): pass +def encode_url_path_segment(value: Any, *, field_name: str = "path parameter") -> str: + """Percent-encode one user-controlled URL path segment. + + ``urllib.parse.quote(..., safe="")`` intentionally leaves RFC 3986 + unreserved characters such as ``.`` unescaped, so reject standalone dot + segments before they can be appended to an upstream URL and normalized by + the HTTP client. + """ + if value is None: + raise ValueError(f"{field_name} is required") + + value_str = str(value) + if value_str == "": + raise ValueError(f"{field_name} is required") + if value_str in {".", ".."}: + raise ValueError(f"{field_name} cannot be a dot path segment") + + return quote(value_str, safe="") + + +def encode_url_path_segments(value: Any, *, field_name: str = "path") -> str: + """Percent-encode a user-controlled URL path made of multiple segments. + + Empty segments are rejected, so leading, trailing, or consecutive slashes + fail closed instead of being normalized by the HTTP client. + """ + if value is None: + raise ValueError(f"{field_name} is required") + + value_str = str(value) + if value_str == "": + raise ValueError(f"{field_name} is required") + + encoded_segments = [] + for segment in value_str.split("/"): + encoded_segments.append(encode_url_path_segment(segment, field_name=field_name)) + + return "/".join(encoded_segments) + + def _is_blocked_ip(addr: str) -> bool: """Return True for any IP not safe to reach from a user-supplied URL. @@ -199,6 +239,47 @@ def validate_url(url: str) -> Tuple[str, str]: return rewritten, host_header +def assert_same_origin(candidate_url: str, expected_url: str) -> None: + """Verify ``candidate_url`` shares scheme, host, and port with ``expected_url``. + + Use when an upstream API returns a URL meant for follow-up requests + (e.g. an async-job polling URL that will be hit with the operator's + API key in the headers). The upstream is trusted because the operator + configured ``api_base``, but the URL it hands back must actually point + back at the same origin or we'd be blindly forwarding credentials + wherever the upstream told us to. + + Hostnames are compared case-insensitively. Default ports are made + explicit (HTTP→80, HTTPS→443) so ``https://api.example.com:443/...`` + and ``https://api.example.com/...`` are treated as the same origin. + + Error messages identify *which* component mismatched but never echo + the operator's ``expected`` host or the candidate's hostname back to + the caller — in the SSRF threat model the caller is the attacker, + and reflecting host info would be a secondary leak of operator + infrastructure details. + """ + candidate = urlparse(candidate_url) + expected = urlparse(expected_url) + + if candidate.scheme not in _ALLOWED_SCHEMES: + raise SSRFError("URL scheme is not allowed") + + if candidate.scheme != expected.scheme: + raise SSRFError("Origin mismatch on scheme") + + candidate_host = _normalize_host(candidate.hostname or "") + expected_host = _normalize_host(expected.hostname or "") + if not candidate_host or candidate_host != expected_host: + raise SSRFError("Origin mismatch on host") + + default_port = 443 if candidate.scheme == "https" else 80 + candidate_port = candidate.port if candidate.port is not None else default_port + expected_port = expected.port if expected.port is not None else default_port + if candidate_port != expected_port: + raise SSRFError("Origin mismatch on port") + + _MAX_REDIRECTS = 10 diff --git a/litellm/llms/anthropic/batches/transformation.py b/litellm/llms/anthropic/batches/transformation.py index 3f03c744efe..fd67a7fbaf1 100644 --- a/litellm/llms/anthropic/batches/transformation.py +++ b/litellm/llms/anthropic/batches/transformation.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cas import httpx from httpx import Headers, Response +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest @@ -122,7 +123,8 @@ class AnthropicBatchesConfig(BaseBatchesConfig): Complete URL for Anthropic batch retrieval: {api_base}/v1/messages/batches/{batch_id} """ api_base = api_base or self.anthropic_model_info.get_api_base(api_base) - return f"{api_base.rstrip('/')}/v1/messages/batches/{batch_id}" + encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id") + return f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}" def transform_retrieve_batch_request( self, diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 31b9bad3395..61ddf801a25 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1553,25 +1553,43 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) data["output_config"] = output_config - def _transform_response_for_json_mode( + def _resolve_json_mode_non_streaming( self, json_mode: Optional[bool], tool_calls: List[ChatCompletionToolCallChunk], - ) -> Optional[LitellmMessage]: - _message: Optional[LitellmMessage] = None - if json_mode is True and len(tool_calls) == 1: - # check if tool name is the default tool name - json_mode_content_str: Optional[str] = None - if ( - "name" in tool_calls[0]["function"] - and tool_calls[0]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME - ): - json_mode_content_str = tool_calls[0]["function"].get("arguments") - if json_mode_content_str is not None: - _message = AnthropicConfig._convert_tool_response_to_message( - tool_calls=tool_calls, - ) - return _message + ) -> Tuple[ + Optional[LitellmMessage], + List[ChatCompletionToolCallChunk], + Optional[str], + ]: + """Strip internal response_format tool calls; merge payload into content when mixed with user tools.""" + if json_mode is not True or not tool_calls: + return None, tool_calls, None + + json_indices = [ + i + for i, t in enumerate(tool_calls) + if t.get("function", {}).get("name") == RESPONSE_FORMAT_TOOL_NAME + ] + if not json_indices: + return None, tool_calls, None + + if len(json_indices) == len(tool_calls): + json_tool = tool_calls[json_indices[0]] + if json_tool.get("function", {}).get("arguments") is None: + return None, tool_calls, None + _message = AnthropicConfig._convert_tool_response_to_message( + tool_calls=[json_tool] + ) + return _message, [], None + + first_json = tool_calls[json_indices[0]] + json_msg = AnthropicConfig._convert_tool_response_to_message([first_json]) + extra_content: Optional[str] = ( + json_msg.content if json_msg is not None else None + ) + filtered_tools = [t for i, t in enumerate(tool_calls) if i not in json_indices] + return None, filtered_tools, extra_content def extract_response_content(self, completion_response: dict) -> Tuple[ str, @@ -1931,19 +1949,27 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_calls, ) + json_mode_message, tool_calls_for_message, json_extra_content = ( + self._resolve_json_mode_non_streaming( + json_mode=json_mode, + tool_calls=tool_calls, + ) + ) + merged_text = text_content or "" + if json_extra_content: + merged_text = ( + merged_text + json_extra_content if merged_text else json_extra_content + ) + _message = litellm.Message( - tool_calls=tool_calls, - content=text_content or None, + tool_calls=tool_calls_for_message, + content=merged_text or None, provider_specific_fields=provider_specific_fields, thinking_blocks=thinking_blocks, reasoning_content=reasoning_content, ) _message.provider_specific_fields = provider_specific_fields - json_mode_message = self._transform_response_for_json_mode( - json_mode=json_mode, - tool_calls=tool_calls, - ) if json_mode_message is not None: completion_response["stop_reason"] = "stop" _message = json_mode_message diff --git a/litellm/llms/anthropic/files/handler.py b/litellm/llms/anthropic/files/handler.py index c56799f30cf..56296df94a1 100644 --- a/litellm/llms/anthropic/files/handler.py +++ b/litellm/llms/anthropic/files/handler.py @@ -9,6 +9,7 @@ import litellm from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.types.llms.openai import ( FileContentRequest, @@ -89,7 +90,10 @@ class AnthropicFilesHandler: raise ValueError("Missing Anthropic API Key") # Construct the Anthropic batch results URL - results_url = f"{api_base.rstrip('/')}/v1/messages/batches/{batch_id}/results" + encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id") + results_url = ( + f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}/results" + ) # Prepare headers headers = { diff --git a/litellm/llms/anthropic/files/transformation.py b/litellm/llms/anthropic/files/transformation.py index aeaab4e57bf..ea9bf00f505 100644 --- a/litellm/llms/anthropic/files/transformation.py +++ b/litellm/llms/anthropic/files/transformation.py @@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optional, Union, cast import httpx from openai.types.file_deleted import FileDeleted +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import ( @@ -185,7 +186,8 @@ class AnthropicFilesConfig(BaseFilesConfig): AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE ) - return f"{api_base.rstrip('/')}/v1/files/{file_id}", {} + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") + return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {} def transform_retrieve_file_response( self, @@ -206,7 +208,8 @@ class AnthropicFilesConfig(BaseFilesConfig): AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE ) - return f"{api_base.rstrip('/')}/v1/files/{file_id}", {} + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") + return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {} def transform_delete_file_response( self, @@ -268,7 +271,8 @@ class AnthropicFilesConfig(BaseFilesConfig): AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE ) - return f"{api_base.rstrip('/')}/v1/files/{file_id}/content", {} + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") + return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}/content", {} def transform_file_content_response( self, diff --git a/litellm/llms/anthropic/skills/transformation.py b/litellm/llms/anthropic/skills/transformation.py index a992d84d459..4ea768b02af 100644 --- a/litellm/llms/anthropic/skills/transformation.py +++ b/litellm/llms/anthropic/skills/transformation.py @@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Tuple import httpx from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.skills.transformation import ( BaseSkillsAPIConfig, LiteLLMLoggingObj, @@ -81,7 +82,8 @@ class AnthropicSkillsConfig(BaseSkillsAPIConfig): api_base = AnthropicModelInfo.get_api_base() if skill_id: - return f"{api_base}/v1/skills/{skill_id}" + encoded_skill_id = encode_url_path_segment(skill_id, field_name="skill_id") + return f"{api_base}/v1/skills/{encoded_skill_id}" return f"{api_base}/v1/{endpoint}" def transform_create_skill_request( diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 61cfd54b565..877a7d3c84a 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -16,6 +16,7 @@ import litellm from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_utils import track_llm_api_timing +from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -898,6 +899,17 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): operation_location_url = response.headers["operation-location"] else: raise AzureOpenAIError(status_code=500, message=response.text) + # Reject polling URLs that don't share an origin with ``api_base``. + # Without this an upstream-controlled or attacker-controlled + # value would receive the operator's Azure API key in the + # request headers below. VERIA-51. + try: + assert_same_origin(operation_location_url, api_base) + except SSRFError as ssrf_err: + raise AzureOpenAIError( + status_code=502, + message=f"Rejected polling URL: {ssrf_err}", + ) response = await async_handler.get( url=operation_location_url, headers=headers, @@ -908,8 +920,13 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): timeout_secs: int = AZURE_OPERATION_POLLING_TIMEOUT start_time = time.time() if "status" not in response.json(): - raise Exception( - "Expected 'status' in response. Got={}".format(response.json()) + # Don't reflect the raw response body — when the polling + # URL points at an internal JSON API (cloud metadata + # service etc.) reflecting it here turns Blind SSRF into + # Full-Read SSRF. VERIA-51. + raise AzureOpenAIError( + status_code=502, + message="Polling response missing 'status' field", ) while response.json()["status"] not in ["succeeded", "failed"]: if time.time() - start_time > timeout_secs: @@ -1009,6 +1026,13 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): operation_location_url = response.headers["operation-location"] else: raise AzureOpenAIError(status_code=500, message=response.text) + try: + assert_same_origin(operation_location_url, api_base) + except SSRFError as ssrf_err: + raise AzureOpenAIError( + status_code=502, + message=f"Rejected polling URL: {ssrf_err}", + ) response = sync_handler.get( url=operation_location_url, headers=headers, @@ -1019,8 +1043,9 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): timeout_secs: int = AZURE_OPERATION_POLLING_TIMEOUT start_time = time.time() if "status" not in response.json(): - raise Exception( - "Expected 'status' in response. Got={}".format(response.json()) + raise AzureOpenAIError( + status_code=502, + message="Polling response missing 'status' field", ) while response.json()["status"] not in ["succeeded", "failed"]: if time.time() - start_time > timeout_secs: diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 76a6d485bc4..ca9293325ff 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -5,6 +5,7 @@ import httpx from openai.types.responses import ResponseReasoningItem from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import * @@ -201,7 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): # Insert the response_id at the end of the path component # Remove trailing slash if present to avoid double slashes path = parsed_url.path.rstrip("/") - new_path = f"{path}/{response_id}" + encoded_response_id = encode_url_path_segment( + response_id, field_name="response_id" + ) + new_path = f"{path}/{encoded_response_id}" # Reconstruct the URL with all original components but with the modified path constructed_url = urlunparse( @@ -322,7 +326,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): # Insert the response_id and /cancel at the end of the path component # Remove trailing slash if present to avoid double slashes path = parsed_url.path.rstrip("/") - new_path = f"{path}/{response_id}/cancel" + encoded_response_id = encode_url_path_segment( + response_id, field_name="response_id" + ) + new_path = f"{path}/{encoded_response_id}/cancel" # Reconstruct the URL with all original components but with the modified path cancel_url = urlunparse( diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index c3cd06ab4de..9bae8abce8e 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -36,6 +36,7 @@ from typing import ( import httpx from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.azure_ai.agents.transformation import ( AzureAIAgentsConfig, AzureAIAgentsError, @@ -75,20 +76,29 @@ class AzureAIAgentsHandler: def _build_messages_url( self, api_base: str, thread_id: str, api_version: str ) -> str: - return f"{api_base}/threads/{thread_id}/messages?api-version={api_version}" + encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id") + return ( + f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}" + ) def _build_runs_url(self, api_base: str, thread_id: str, api_version: str) -> str: - return f"{api_base}/threads/{thread_id}/runs?api-version={api_version}" + encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id") + return f"{api_base}/threads/{encoded_thread_id}/runs?api-version={api_version}" def _build_run_status_url( self, api_base: str, thread_id: str, run_id: str, api_version: str ) -> str: - return f"{api_base}/threads/{thread_id}/runs/{run_id}?api-version={api_version}" + encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id") + encoded_run_id = encode_url_path_segment(run_id, field_name="run_id") + return f"{api_base}/threads/{encoded_thread_id}/runs/{encoded_run_id}?api-version={api_version}" def _build_list_messages_url( self, api_base: str, thread_id: str, api_version: str ) -> str: - return f"{api_base}/threads/{thread_id}/messages?api-version={api_version}" + encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id") + return ( + f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}" + ) def _build_create_thread_and_run_url(self, api_base: str, api_version: str) -> str: """URL for the create-thread-and-run endpoint (supports streaming).""" diff --git a/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py index 76c247aea81..d4144a75718 100644 --- a/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py +++ b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py @@ -17,11 +17,13 @@ from urllib.parse import quote import httpx from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin from litellm.constants import ( AZURE_DOCUMENT_INTELLIGENCE_API_VERSION, AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI, AZURE_OPERATION_POLLING_TIMEOUT, ) +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.ocr.transformation import ( BaseOCRConfig, DocumentType, @@ -217,11 +219,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): if "/" in model: # Extract the last part after the last slash model_id = model.split("/")[-1] + encoded_model_id = encode_url_path_segment(model_id, field_name="model_id") # Azure Document Intelligence analyze endpoint # Note: API version 2024-11-30+ uses /documentintelligence/ (not /formrecognizer/) url = ( - f"{api_base}/documentintelligence/documentModels/{model_id}:analyze" + f"{api_base}/documentintelligence/documentModels/{encoded_model_id}:analyze" f"?api-version={AZURE_DOCUMENT_INTELLIGENCE_API_VERSION}" ) @@ -599,6 +602,16 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): "Azure Document Intelligence returned 202 but no Operation-Location header found" ) + # Reject cross-origin polling URLs — the auth headers + # below would otherwise leak to whatever URL the upstream + # (or an attacker-controlled upstream) returns. VERIA-51. + try: + assert_same_origin(operation_url, str(raw_response.request.url)) + except SSRFError as ssrf_err: + raise ValueError( + f"Azure Document Intelligence: rejected polling URL ({ssrf_err})" + ) + # Get headers for polling (need auth) poll_headers = { "Ocp-Apim-Subscription-Key": raw_response.request.headers.get( @@ -711,6 +724,14 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): "Azure Document Intelligence returned 202 but no Operation-Location header found" ) + # Reject cross-origin polling URLs (see sync path). VERIA-51. + try: + assert_same_origin(operation_url, str(raw_response.request.url)) + except SSRFError as ssrf_err: + raise ValueError( + f"Azure Document Intelligence: rejected polling URL ({ssrf_err})" + ) + # Get headers for polling (need auth) poll_headers = { "Ocp-Apim-Subscription-Key": raw_response.request.headers.get( diff --git a/litellm/llms/base_llm/rerank/transformation.py b/litellm/llms/base_llm/rerank/transformation.py index 7874201f7f0..166f876ba04 100644 --- a/litellm/llms/base_llm/rerank/transformation.py +++ b/litellm/llms/base_llm/rerank/transformation.py @@ -33,6 +33,7 @@ class BaseRerankConfig(ABC): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: return {} diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 4e3521b119e..dae60948a58 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -1,6 +1,7 @@ import hashlib import json import os +import re import urllib.parse from datetime import datetime from typing import ( @@ -37,6 +38,11 @@ else: AWSPreparedRequest = Any +# Real AWS region names are lowercase letters, digits, and hyphens +# (e.g. "us-east-1", "eu-west-2", "us-gov-west-1", "cn-north-1"). +_VALID_AWS_REGION_PATTERN = re.compile(r"\A[a-z0-9-]+\Z") + + class Boto3CredentialsInfo(BaseModel): credentials: Credentials aws_region_name: str @@ -284,6 +290,9 @@ class BaseAWSLLM: if not region: # Check if region is empty return None + if not _VALID_AWS_REGION_PATTERN.match(region): + return None + return region except Exception: # Catch any unexpected errors and return None @@ -481,6 +490,7 @@ class BaseAWSLLM: str: The AWS region name """ aws_region_name = optional_params.get("aws_region_name", None) + self._validate_aws_region_name(aws_region_name) ### SET REGION NAME ### if aws_region_name is None: # check model arn # @@ -519,8 +529,25 @@ class BaseAWSLLM: except Exception: aws_region_name = "us-west-2" + self._validate_aws_region_name(aws_region_name) return aws_region_name + @staticmethod + def _validate_aws_region_name(aws_region_name: Optional[str]) -> None: + """ + Validate that an AWS region name conforms to the expected format + (lowercase alphanumerics and hyphens). Raises ValueError otherwise. + """ + if aws_region_name is None: + return + if not isinstance(aws_region_name, str) or not _VALID_AWS_REGION_PATTERN.match( + aws_region_name + ): + raise ValueError( + f"Invalid AWS region format: {aws_region_name!r}. " + "Region names must contain only lowercase letters, digits, and hyphens." + ) + def get_aws_region_name_for_non_llm_api_calls( self, aws_region_name: Optional[str] = None, @@ -532,6 +559,7 @@ class BaseAWSLLM: For non-llm api calls eg. Guardrails, Vector Stores we just need to check the dynamic param or env vars. """ + self._validate_aws_region_name(aws_region_name) if aws_region_name is None: # check env # litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) @@ -549,6 +577,8 @@ class BaseAWSLLM: if aws_region_name is None: aws_region_name = "us-west-2" + + self._validate_aws_region_name(aws_region_name) return aws_region_name @staticmethod diff --git a/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/litellm/llms/bedrock/chat/invoke_agent/transformation.py index 2c7135f4d83..4c667b0ce39 100644 --- a/litellm/llms/bedrock/chat/invoke_agent/transformation.py +++ b/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -12,6 +12,7 @@ import httpx from litellm._logging import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) @@ -97,8 +98,15 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): agent_id, agent_alias_id = self._get_agent_id_and_alias_id(model) session_id = self._get_session_id(optional_params) + encoded_agent_id = encode_url_path_segment(agent_id, field_name="agent_id") + encoded_agent_alias_id = encode_url_path_segment( + agent_alias_id, field_name="agent_alias_id" + ) + encoded_session_id = encode_url_path_segment( + session_id, field_name="session_id" + ) - endpoint_url = f"{endpoint_url}/agents/{agent_id}/agentAliases/{agent_alias_id}/sessions/{session_id}/text" + endpoint_url = f"{endpoint_url}/agents/{encoded_agent_id}/agentAliases/{encoded_agent_alias_id}/sessions/{encoded_session_id}/text" return endpoint_url diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index a37af131625..c967fd334bc 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -201,13 +201,14 @@ class BedrockCountTokensConfig(BaseAWSLLM): # Remove bedrock/ prefix if present if model_id.startswith("bedrock/"): model_id = model_id[8:] # Remove "bedrock/" prefix + encoded_model_id = self.encode_model_id(model_id=model_id) base_url, _ = self.get_runtime_endpoint( api_base=api_base, aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, aws_region_name=aws_region_name, ) - endpoint = f"{base_url}/model/{model_id}/count-tokens" + endpoint = f"{base_url}/model/{encoded_model_id}/count-tokens" return endpoint diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index f028503c6a2..ec20d76102b 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -5,6 +5,7 @@ from urllib.parse import urlparse import httpx from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.types.integrations.rag.bedrock_knowledgebase import ( @@ -209,7 +210,10 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): if isinstance(query, list): query = " ".join(query) - url = f"{api_base}/{vector_store_id}/retrieve" + encoded_vector_store_id = encode_url_path_segment( + vector_store_id, field_name="vector_store_id" + ) + url = f"{api_base}/{encoded_vector_store_id}/retrieve" request_body: Dict[str, Any] = { "retrievalQuery": BedrockKBRetrievalQuery(text=query), diff --git a/litellm/llms/black_forest_labs/image_edit/handler.py b/litellm/llms/black_forest_labs/image_edit/handler.py index dea2683a049..f5784e08367 100644 --- a/litellm/llms/black_forest_labs/image_edit/handler.py +++ b/litellm/llms/black_forest_labs/image_edit/handler.py @@ -15,6 +15,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -331,6 +332,17 @@ class BlackForestLabsImageEdit: message="No polling_url in BFL response", ) + # Reject cross-origin polling URLs — the ``x-key`` auth header + # would otherwise leak to whatever URL the upstream returns. + # VERIA-51. + try: + assert_same_origin(polling_url, str(initial_response.request.url)) + except SSRFError as ssrf_err: + raise BlackForestLabsError( + status_code=502, + message=f"Rejected polling URL: {ssrf_err}", + ) + # Get just the auth header for polling polling_headers = {"x-key": headers.get("x-key", "")} @@ -416,6 +428,17 @@ class BlackForestLabsImageEdit: message="No polling_url in BFL response", ) + # Reject cross-origin polling URLs — the ``x-key`` auth header + # would otherwise leak to whatever URL the upstream returns. + # VERIA-51. + try: + assert_same_origin(polling_url, str(initial_response.request.url)) + except SSRFError as ssrf_err: + raise BlackForestLabsError( + status_code=502, + message=f"Rejected polling URL: {ssrf_err}", + ) + # Get just the auth header for polling polling_headers = {"x-key": headers.get("x-key", "")} diff --git a/litellm/llms/black_forest_labs/image_generation/handler.py b/litellm/llms/black_forest_labs/image_generation/handler.py index 5a1d885e527..8af4a236fd4 100644 --- a/litellm/llms/black_forest_labs/image_generation/handler.py +++ b/litellm/llms/black_forest_labs/image_generation/handler.py @@ -15,6 +15,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -317,6 +318,17 @@ class BlackForestLabsImageGeneration: message="No polling_url in BFL response", ) + # Reject cross-origin polling URLs — the ``x-key`` auth header + # would otherwise leak to whatever URL the upstream returns. + # VERIA-51. + try: + assert_same_origin(polling_url, str(initial_response.request.url)) + except SSRFError as ssrf_err: + raise BlackForestLabsError( + status_code=502, + message=f"Rejected polling URL: {ssrf_err}", + ) + # Get just the auth header for polling polling_headers = {"x-key": headers.get("x-key", "")} @@ -402,6 +414,17 @@ class BlackForestLabsImageGeneration: message="No polling_url in BFL response", ) + # Reject cross-origin polling URLs — the ``x-key`` auth header + # would otherwise leak to whatever URL the upstream returns. + # VERIA-51. + try: + assert_same_origin(polling_url, str(initial_response.request.url)) + except SSRFError as ssrf_err: + raise BlackForestLabsError( + status_code=502, + message=f"Rejected polling URL: {ssrf_err}", + ) + # Get just the auth header for polling polling_headers = {"x-key": headers.get("x-key", "")} diff --git a/litellm/llms/bytez/chat/transformation.py b/litellm/llms/bytez/chat/transformation.py index a72f732a303..5b08670f9f2 100644 --- a/litellm/llms/bytez/chat/transformation.py +++ b/litellm/llms/bytez/chat/transformation.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union import httpx +from litellm.litellm_core_utils.url_utils import encode_url_path_segments from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.logging_utils import track_llm_api_timing from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -149,7 +150,8 @@ class BytezChatConfig(BaseConfig): litellm_params: dict, stream: Optional[bool] = None, ) -> str: - return f"{API_BASE}/{model}" + encoded_model = encode_url_path_segments(model, field_name="model") + return f"{API_BASE}/{encoded_model}" def transform_request( self, diff --git a/litellm/llms/cloudflare/chat/transformation.py b/litellm/llms/cloudflare/chat/transformation.py index 9e59782bf73..b9e219f5cbc 100644 --- a/litellm/llms/cloudflare/chat/transformation.py +++ b/litellm/llms/cloudflare/chat/transformation.py @@ -5,6 +5,7 @@ from typing import AsyncIterator, Iterator, List, Optional, Union import httpx import litellm +from litellm.litellm_core_utils.url_utils import encode_url_path_segments from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import ( BaseConfig, @@ -89,7 +90,8 @@ class CloudflareChatConfig(BaseConfig): api_base = ( f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/" ) - return api_base + model + encoded_model = encode_url_path_segments(model, field_name="model") + return api_base + encoded_model def get_supported_openai_params(self, model: str) -> List[str]: return [ diff --git a/litellm/llms/cohere/rerank/transformation.py b/litellm/llms/cohere/rerank/transformation.py index 531b94d1805..64ae8e8ffa7 100644 --- a/litellm/llms/cohere/rerank/transformation.py +++ b/litellm/llms/cohere/rerank/transformation.py @@ -111,6 +111,7 @@ class CohereRerankConfig(BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: if "query" not in optional_rerank_params: raise ValueError("query is required for Cohere rerank") diff --git a/litellm/llms/cohere/rerank_v2/transformation.py b/litellm/llms/cohere/rerank_v2/transformation.py index 60d22ff4be0..4c800d6455d 100644 --- a/litellm/llms/cohere/rerank_v2/transformation.py +++ b/litellm/llms/cohere/rerank_v2/transformation.py @@ -71,6 +71,7 @@ class CohereRerankV2Config(CohereRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: if "query" not in optional_rerank_params: raise ValueError("query is required for Cohere rerank") diff --git a/litellm/llms/custom_httpx/container_handler.py b/litellm/llms/custom_httpx/container_handler.py index afdd7bc6a8b..599cd705ebf 100644 --- a/litellm/llms/custom_httpx/container_handler.py +++ b/litellm/llms/custom_httpx/container_handler.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Type, Union import httpx import litellm +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -72,7 +73,8 @@ def _build_url( # Substitute path parameters for param, value in path_params.items(): - path_template = path_template.replace(f"{{{param}}}", value) + encoded_value = encode_url_path_segment(value, field_name=param) + path_template = path_template.replace(f"{{{param}}}", encoded_value) # Parse the api_base to extract existing query params parsed_base = httpx.URL(api_base) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a34b73b5313..0c4816fcda2 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -26,6 +26,7 @@ from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -1007,6 +1008,7 @@ class BaseLLMHTTPHandler: api_key: Optional[str] = None, api_base: Optional[str] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + litellm_params: Optional[Dict[str, Any]] = None, ) -> RerankResponse: # get config from model, custom llm provider headers = provider_config.validate_environment( @@ -1026,6 +1028,7 @@ class BaseLLMHTTPHandler: model=model, optional_rerank_params=optional_rerank_params, headers=headers, + litellm_params=litellm_params, ) ## LOGGING @@ -2535,10 +2538,16 @@ class BaseLLMHTTPHandler: }, ) + delete_kwargs: Dict[str, Any] = { + "url": url, + "headers": headers, + "timeout": timeout, + } + if data: + delete_kwargs["json"] = data + try: - response = await async_httpx_client.delete( - url=url, headers=headers, json=data, timeout=timeout - ) + response = await async_httpx_client.delete(**delete_kwargs) except Exception as e: raise self._handle_error( @@ -2619,10 +2628,16 @@ class BaseLLMHTTPHandler: }, ) + delete_kwargs: Dict[str, Any] = { + "url": url, + "headers": headers, + "timeout": timeout, + } + if data: + delete_kwargs["json"] = data + try: - response = sync_httpx_client.delete( - url=url, headers=headers, json=data, timeout=timeout - ) + response = sync_httpx_client.delete(**delete_kwargs) except Exception as e: raise self._handle_error( @@ -8934,7 +8949,10 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url = f"{api_base}/{vector_store_id}" + encoded_vector_store_id = encode_url_path_segment( + vector_store_id, field_name="vector_store_id" + ) + url = f"{api_base}/{encoded_vector_store_id}" logging_obj.pre_call( input="", @@ -9001,7 +9019,10 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url = f"{api_base}/{vector_store_id}" + encoded_vector_store_id = encode_url_path_segment( + vector_store_id, field_name="vector_store_id" + ) + url = f"{api_base}/{encoded_vector_store_id}" logging_obj.pre_call( input="", @@ -9200,7 +9221,10 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url = f"{api_base}/{vector_store_id}" + encoded_vector_store_id = encode_url_path_segment( + vector_store_id, field_name="vector_store_id" + ) + url = f"{api_base}/{encoded_vector_store_id}" request_body: Dict[str, Any] = dict(vector_store_update_optional_params) @@ -9283,7 +9307,10 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url = f"{api_base}/{vector_store_id}" + encoded_vector_store_id = encode_url_path_segment( + vector_store_id, field_name="vector_store_id" + ) + url = f"{api_base}/{encoded_vector_store_id}" request_body: Dict[str, Any] = dict(vector_store_update_optional_params) @@ -9349,7 +9376,10 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url = f"{api_base}/{vector_store_id}" + encoded_vector_store_id = encode_url_path_segment( + vector_store_id, field_name="vector_store_id" + ) + url = f"{api_base}/{encoded_vector_store_id}" logging_obj.pre_call( input="", @@ -9414,7 +9444,10 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url = f"{api_base}/{vector_store_id}" + encoded_vector_store_id = encode_url_path_segment( + vector_store_id, field_name="vector_store_id" + ) + url = f"{api_base}/{encoded_vector_store_id}" logging_obj.pre_call( input="", diff --git a/litellm/llms/deepinfra/rerank/transformation.py b/litellm/llms/deepinfra/rerank/transformation.py index 71e300d258c..276735f4758 100644 --- a/litellm/llms/deepinfra/rerank/transformation.py +++ b/litellm/llms/deepinfra/rerank/transformation.py @@ -132,6 +132,7 @@ class DeepinfraRerankConfig(BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: # Convert OptionalRerankParams to dict as expected by parent class if optional_rerank_params is None: diff --git a/litellm/llms/elevenlabs/text_to_speech/transformation.py b/litellm/llms/elevenlabs/text_to_speech/transformation.py index 4dac2b8ba92..6a59911701b 100644 --- a/litellm/llms/elevenlabs/text_to_speech/transformation.py +++ b/litellm/llms/elevenlabs/text_to_speech/transformation.py @@ -11,13 +11,14 @@ import httpx from httpx import Headers import litellm -from litellm.types.utils import all_litellm_params +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, TextToSpeechRequestData, ) from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import all_litellm_params from ..common_utils import ElevenLabsException @@ -321,7 +322,8 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig): "ElevenLabs voice_id is required. Pass `voice` when calling `litellm.speech()`." ) - url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{voice_id}" + encoded_voice_id = encode_url_path_segment(voice_id, field_name="voice_id") + url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{encoded_voice_id}" query_params = litellm_params.get(self.ELEVENLABS_QUERY_PARAMS_KEY, {}) if query_params: diff --git a/litellm/llms/fireworks_ai/rerank/transformation.py b/litellm/llms/fireworks_ai/rerank/transformation.py index eb92399a058..4a7b64b9b77 100644 --- a/litellm/llms/fireworks_ai/rerank/transformation.py +++ b/litellm/llms/fireworks_ai/rerank/transformation.py @@ -127,6 +127,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: """ Transform request to Fireworks AI rerank format diff --git a/litellm/llms/gemini/files/transformation.py b/litellm/llms/gemini/files/transformation.py index 401d7bb9f48..63a383ebd3d 100644 --- a/litellm/llms/gemini/files/transformation.py +++ b/litellm/llms/gemini/files/transformation.py @@ -12,6 +12,7 @@ import httpx from openai.types.file_deleted import FileDeleted from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data from litellm.llms.base_llm.files.transformation import ( BaseFilesConfig, @@ -258,10 +259,14 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): normalized_file_id = file_id normalized_file_id = normalized_file_id.strip("/") - if not normalized_file_id.startswith("files/"): - normalized_file_id = f"files/{normalized_file_id}" + if normalized_file_id.startswith("files/"): + normalized_file_id = normalized_file_id.removeprefix("files/") - return normalized_file_id + encoded_file_id = encode_url_path_segment( + normalized_file_id, field_name="file_id" + ) + + return f"files/{encoded_file_id}" def transform_retrieve_file_response( self, @@ -337,13 +342,8 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): if not api_key: raise ValueError("api_key is required") - # Extract file name from URI if full URI is provided - # file_id could be "files/abc123" or "https://generativelanguage.googleapis.com/v1beta/files/abc123" - if file_id.startswith("http"): - # Extract the file path from full URI - file_name = file_id.split("/v1beta/")[-1] - else: - file_name = file_id if file_id.startswith("files/") else f"files/{file_id}" + # Normalize and encode the file name before interpolating it into the URL. + file_name = self._normalize_gemini_file_id(file_id) # Construct the delete URL url = f"{api_base}/v1beta/{file_name}" diff --git a/litellm/llms/gemini/interactions/transformation.py b/litellm/llms/gemini/interactions/transformation.py index c34da83cb8f..593cbf7c2cf 100644 --- a/litellm/llms/gemini/interactions/transformation.py +++ b/litellm/llms/gemini/interactions/transformation.py @@ -15,6 +15,7 @@ import httpx from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo from litellm.types.interactions import ( @@ -205,8 +206,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): resolved_api_base = GeminiModelInfo.get_api_base(api_base) if not GeminiModelInfo.get_api_key(litellm_params.api_key): raise ValueError("Google API key is required") + encoded_interaction_id = encode_url_path_segment( + interaction_id, field_name="interaction_id" + ) return ( - f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}", + f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}", {}, ) @@ -238,8 +242,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): resolved_api_base = GeminiModelInfo.get_api_base(api_base) if not GeminiModelInfo.get_api_key(litellm_params.api_key): raise ValueError("Google API key is required") + encoded_interaction_id = encode_url_path_segment( + interaction_id, field_name="interaction_id" + ) return ( - f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}", + f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}", {}, ) @@ -268,8 +275,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): resolved_api_base = GeminiModelInfo.get_api_base(api_base) if not GeminiModelInfo.get_api_key(litellm_params.api_key): raise ValueError("Google API key is required") + encoded_interaction_id = encode_url_path_segment( + interaction_id, field_name="interaction_id" + ) return ( - f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}:cancel", + f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}:cancel", {}, ) diff --git a/litellm/llms/hosted_vllm/embedding/README.md b/litellm/llms/hosted_vllm/embedding/README.md index f82b3c77a6e..2c58e16fc23 100644 --- a/litellm/llms/hosted_vllm/embedding/README.md +++ b/litellm/llms/hosted_vllm/embedding/README.md @@ -2,4 +2,15 @@ No transformation is required for hosted_vllm embedding. VLLM is a superset of OpenAI's `embedding` endpoint. -To pass provider-specific parameters, see [this](https://docs.litellm.ai/docs/completion/provider_specific_params) \ No newline at end of file +## `encoding_format` + +For OpenAI-compatible embedding calls (including `openai/...` with a custom `api_base` pointing at vLLM), LiteLLM resolves `encoding_format` when it is not set on the request: + +1. Explicit value on the embedding call (`encoding_format=...`). +2. Model config (`litellm_params.encoding_format` on the proxy `model_list` entry). +3. Environment variable `LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT` (e.g. in `.env` or container env). +4. Default **`float`**. + +That avoids forwarding `encoding_format=None` to the provider/SDK where some servers behave poorly. + +To pass provider-specific parameters, see [provider-specific params](https://docs.litellm.ai/docs/completion/provider_specific_params). \ No newline at end of file diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 8066e53afc7..60b6dc7d23d 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -121,6 +121,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: if "query" not in optional_rerank_params: raise ValueError("query is required for Hosted VLLM rerank") diff --git a/litellm/llms/huggingface/rerank/transformation.py b/litellm/llms/huggingface/rerank/transformation.py index 3f83b8e422d..2c847b617ef 100644 --- a/litellm/llms/huggingface/rerank/transformation.py +++ b/litellm/llms/huggingface/rerank/transformation.py @@ -146,6 +146,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig): model: str, optional_rerank_params: Union[OptionalRerankParams, dict], headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: if "query" not in optional_rerank_params: raise ValueError("query is required for HuggingFace rerank") diff --git a/litellm/llms/jina_ai/rerank/transformation.py b/litellm/llms/jina_ai/rerank/transformation.py index 48d876f8ea2..ad4416925a6 100644 --- a/litellm/llms/jina_ai/rerank/transformation.py +++ b/litellm/llms/jina_ai/rerank/transformation.py @@ -74,7 +74,11 @@ class JinaAIRerankConfig(BaseRerankConfig): return cleaned_base def transform_rerank_request( - self, model: str, optional_rerank_params: Dict, headers: Dict + self, + model: str, + optional_rerank_params: Dict, + headers: Dict, + litellm_params: Optional[dict] = None, ) -> Dict: return {"model": model, **optional_rerank_params} diff --git a/litellm/llms/manus/files/transformation.py b/litellm/llms/manus/files/transformation.py index 3381a5327e8..34166161390 100644 --- a/litellm/llms/manus/files/transformation.py +++ b/litellm/llms/manus/files/transformation.py @@ -18,6 +18,7 @@ from openai.types.file_deleted import FileDeleted import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import ( @@ -306,7 +307,8 @@ class ManusFilesConfig(BaseFilesConfig): optional_params=optional_params, litellm_params=litellm_params, ) - return f"{api_base}/{file_id}", {} + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") + return f"{api_base}/{encoded_file_id}", {} def transform_retrieve_file_response( self, @@ -336,7 +338,8 @@ class ManusFilesConfig(BaseFilesConfig): optional_params=optional_params, litellm_params=litellm_params, ) - return f"{api_base}/{file_id}", {} + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") + return f"{api_base}/{encoded_file_id}", {} def transform_delete_file_response( self, @@ -422,7 +425,8 @@ class ManusFilesConfig(BaseFilesConfig): optional_params=optional_params, litellm_params=litellm_params, ) - return f"{api_base}/{file_id}/content", {} + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") + return f"{api_base}/{encoded_file_id}/content", {} def transform_file_content_response( self, diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index 510c41304a8..b3a0073a5c2 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -6,6 +6,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( _safe_convert_created_field, ) @@ -270,7 +271,10 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): Reference: https://open.manus.im/docs/openai-compatibility """ - url = f"{api_base}/{response_id}" + encoded_response_id = encode_url_path_segment( + response_id, field_name="response_id" + ) + url = f"{api_base}/{encoded_response_id}" data: Dict = {} return url, data diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py index af78cd8dbda..867f6d4b1f5 100644 --- a/litellm/llms/milvus/vector_stores/transformation.py +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -25,7 +25,6 @@ else: LiteLLMLoggingObj = Any MILVUS_OPTIONAL_PARAMS = { - "dbName", "annsField", "limit", "filter", @@ -33,7 +32,6 @@ MILVUS_OPTIONAL_PARAMS = { "groupingField", "outputFields", "searchParams", - "partitionNames", "consistencyLevel", } @@ -173,13 +171,21 @@ class MilvusVectorStoreConfig(BaseVectorStoreConfig): url = f"{api_base}/v2/vectordb/entities/search" # Build the request body for Azure AI Search with vector search - request_body = { + request_body: Dict[str, Any] = { "collectionName": index_name, "data": [query_vector], "annsField": "book_intro_vector", **vector_store_search_optional_params, } + db_name = litellm_params.get("milvus_db_name") + if db_name: + request_body["dbName"] = db_name + + partition_names = litellm_params.get("milvus_partition_names") + if partition_names: + request_body["partitionNames"] = partition_names + ######################################################### # Update logging object with details of the request ######################################################### diff --git a/litellm/llms/nvidia_nim/rerank/ranking_transformation.py b/litellm/llms/nvidia_nim/rerank/ranking_transformation.py index 757d874bf31..b9a46b8ac2b 100644 --- a/litellm/llms/nvidia_nim/rerank/ranking_transformation.py +++ b/litellm/llms/nvidia_nim/rerank/ranking_transformation.py @@ -66,6 +66,7 @@ class NvidiaNimRankingConfig(NvidiaNimRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: """ Transform request, using clean model name without 'ranking/' prefix. @@ -75,4 +76,5 @@ class NvidiaNimRankingConfig(NvidiaNimRerankConfig): model=clean_model, optional_rerank_params=optional_rerank_params, headers=headers, + litellm_params=litellm_params, ) diff --git a/litellm/llms/nvidia_nim/rerank/transformation.py b/litellm/llms/nvidia_nim/rerank/transformation.py index bd5abac60c8..fc317293acc 100644 --- a/litellm/llms/nvidia_nim/rerank/transformation.py +++ b/litellm/llms/nvidia_nim/rerank/transformation.py @@ -177,6 +177,7 @@ class NvidiaNimRerankConfig(BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: """ Transform request to Nvidia NIM format. diff --git a/litellm/llms/openai/containers/transformation.py b/litellm/llms/openai/containers/transformation.py index 955b9f760d1..7f874ffd3b1 100644 --- a/litellm/llms/openai/containers/transformation.py +++ b/litellm/llms/openai/containers/transformation.py @@ -6,6 +6,7 @@ import litellm from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.secret_managers.main import get_secret_str from litellm.types.containers.main import ( ContainerCreateOptionalRequestParams, @@ -198,7 +199,10 @@ class OpenAIContainerConfig(BaseContainerConfig): ) -> Tuple[str, Dict]: """Transform the OpenAI container retrieve request.""" # For container retrieve, we just need to construct the URL - url = join_container_api_base_path(api_base, f"/{container_id}") + encoded_container_id = encode_url_path_segment( + container_id, field_name="container_id" + ) + url = join_container_api_base_path(api_base, f"/{encoded_container_id}") # No additional data needed for GET request data: Dict[str, Any] = {} @@ -230,7 +234,10 @@ class OpenAIContainerConfig(BaseContainerConfig): - DELETE /v1/containers/{container_id} """ # Construct the URL for container delete - url = join_container_api_base_path(api_base, f"/{container_id}") + encoded_container_id = encode_url_path_segment( + container_id, field_name="container_id" + ) + url = join_container_api_base_path(api_base, f"/{encoded_container_id}") # No data needed for DELETE request data: Dict[str, Any] = {} @@ -267,7 +274,10 @@ class OpenAIContainerConfig(BaseContainerConfig): - GET /v1/containers/{container_id}/files """ # Construct the URL for container files - url = join_container_api_base_path(api_base, f"/{container_id}/files") + encoded_container_id = encode_url_path_segment( + container_id, field_name="container_id" + ) + url = join_container_api_base_path(api_base, f"/{encoded_container_id}/files") # Prepare query parameters params: Dict[str, Any] = {} @@ -311,8 +321,12 @@ class OpenAIContainerConfig(BaseContainerConfig): - GET /v1/containers/{container_id}/files/{file_id}/content """ # Construct the URL for container file content + encoded_container_id = encode_url_path_segment( + container_id, field_name="container_id" + ) + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") url = join_container_api_base_path( - api_base, f"/{container_id}/files/{file_id}/content" + api_base, f"/{encoded_container_id}/files/{encoded_file_id}/content" ) # No query parameters needed diff --git a/litellm/llms/openai/evals/transformation.py b/litellm/llms/openai/evals/transformation.py index c24dbf8637a..66537e56a6f 100644 --- a/litellm/llms/openai/evals/transformation.py +++ b/litellm/llms/openai/evals/transformation.py @@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Tuple import httpx from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.evals.transformation import ( BaseEvalsAPIConfig, LiteLLMLoggingObj, @@ -76,7 +77,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): api_base = "https://api.openai.com" if eval_id: - return f"{api_base}/v1/evals/{eval_id}" + encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id") + return f"{api_base}/v1/evals/{encoded_eval_id}" return f"{api_base}/v1/{endpoint}" def transform_create_eval_request( @@ -276,7 +278,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): if litellm_params and litellm_params.api_base: api_base = litellm_params.api_base - url = f"{api_base}/v1/evals/{eval_id}/runs" + encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id") + url = f"{api_base}/v1/evals/{encoded_eval_id}/runs" # Build request body request_body = {k: v for k, v in create_request.items() if v is not None} @@ -310,7 +313,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): if litellm_params and litellm_params.api_base: api_base = litellm_params.api_base - url = f"{api_base}/v1/evals/{eval_id}/runs" + encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id") + url = f"{api_base}/v1/evals/{encoded_eval_id}/runs" # Build query parameters query_params: Dict[str, Any] = {} @@ -350,7 +354,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): headers: dict, ) -> Tuple[str, Dict]: """Transform get run request for OpenAI""" - url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}" + encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id") + encoded_run_id = encode_url_path_segment(run_id, field_name="run_id") + url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}" verbose_logger.debug("Get run request - URL: %s", url) @@ -376,7 +382,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): headers: dict, ) -> Tuple[str, Dict, Dict]: """Transform cancel run request for OpenAI""" - url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}/cancel" + encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id") + encoded_run_id = encode_url_path_segment(run_id, field_name="run_id") + url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}/cancel" # Empty body for cancel request request_body: Dict[str, Any] = {} @@ -405,7 +413,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): headers: dict, ) -> Tuple[str, Dict, Dict]: """Transform delete run request for OpenAI""" - url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}" + encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id") + encoded_run_id = encode_url_path_segment(run_id, field_name="run_id") + url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}" # Empty body for delete request request_body: Dict[str, Any] = {} diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 87c502032cc..b7d5340d8d4 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -7,6 +7,7 @@ from pydantic import BaseModel, ValidationError import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( _safe_convert_created_field, ) @@ -421,7 +422,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): OpenAI API expects the following request - DELETE /v1/responses/{response_id} """ - url = f"{api_base}/{response_id}" + encoded_response_id = encode_url_path_segment( + response_id, field_name="response_id" + ) + url = f"{api_base}/{encoded_response_id}" data: Dict = {} return url, data @@ -457,7 +461,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): OpenAI API expects the following request - GET /v1/responses/{response_id} """ - url = f"{api_base}/{response_id}" + encoded_response_id = encode_url_path_segment( + response_id, field_name="response_id" + ) + url = f"{api_base}/{encoded_response_id}" data: Dict = {} return url, data @@ -498,7 +505,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): limit: int = 20, order: Literal["asc", "desc"] = "desc", ) -> Tuple[str, Dict]: - url = f"{api_base}/{response_id}/input_items" + encoded_response_id = encode_url_path_segment( + response_id, field_name="response_id" + ) + url = f"{api_base}/{encoded_response_id}/input_items" params: Dict[str, Any] = {} if after is not None: params["after"] = after @@ -540,7 +550,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): OpenAI API expects the following request - POST /v1/responses/{response_id}/cancel """ - url = f"{api_base}/{response_id}/cancel" + encoded_response_id = encode_url_path_segment( + response_id, field_name="response_id" + ) + url = f"{api_base}/{encoded_response_id}/cancel" data: Dict = {} return url, data diff --git a/litellm/llms/openai/vector_store_files/transformation.py b/litellm/llms/openai/vector_store_files/transformation.py index cd5f10251bb..52202f57fd3 100644 --- a/litellm/llms/openai/vector_store_files/transformation.py +++ b/litellm/llms/openai/vector_store_files/transformation.py @@ -3,6 +3,7 @@ from typing import Any, Dict, Optional, Tuple, cast import httpx import litellm +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.vector_store_files.transformation import ( BaseVectorStoreFilesConfig, ) @@ -98,7 +99,10 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig): or "https://api.openai.com/v1" ) base_url = base_url.rstrip("/") - return f"{base_url}/vector_stores/{vector_store_id}/files" + encoded_vector_store_id = encode_url_path_segment( + vector_store_id, field_name="vector_store_id" + ) + return f"{base_url}/vector_stores/{encoded_vector_store_id}/files" def transform_create_vector_store_file_request( self, @@ -163,7 +167,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig): file_id: str, api_base: str, ) -> Tuple[str, Dict[str, Any]]: - return f"{api_base}/{file_id}", {} + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") + return f"{api_base}/{encoded_file_id}", {} def transform_retrieve_vector_store_file_response( self, @@ -186,7 +191,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig): file_id: str, api_base: str, ) -> Tuple[str, Dict[str, Any]]: - return f"{api_base}/{file_id}/content", {} + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") + return f"{api_base}/{encoded_file_id}/content", {} def transform_retrieve_vector_store_file_content_response( self, @@ -218,7 +224,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig): payload["attributes"] = filtered_attributes else: payload.pop("attributes", None) - return f"{api_base}/{file_id}", payload + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") + return f"{api_base}/{encoded_file_id}", payload def transform_update_vector_store_file_response( self, @@ -241,7 +248,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig): file_id: str, api_base: str, ) -> Tuple[str, Dict[str, Any]]: - return f"{api_base}/{file_id}", {} + encoded_file_id = encode_url_path_segment(file_id, field_name="file_id") + return f"{api_base}/{encoded_file_id}", {} def transform_delete_vector_store_file_response( self, diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py index 2c11d137480..bd095a0a1b7 100644 --- a/litellm/llms/openai/vector_stores/transformation.py +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast import httpx import litellm +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams @@ -108,7 +109,10 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: - url = f"{api_base}/{vector_store_id}/search" + encoded_vector_store_id = encode_url_path_segment( + vector_store_id, field_name="vector_store_id" + ) + url = f"{api_base}/{encoded_vector_store_id}/search" typed_request_body = VectorStoreSearchRequest( query=query, filters=vector_store_search_optional_params.get("filters", None), diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 61baa56949c..2d165a7d7df 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -1,11 +1,13 @@ import mimetypes from io import BufferedReader, BytesIO from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast +from urllib.parse import quote import httpx from httpx._types import RequestFiles import litellm +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.openai.image_edit.transformation import ImageEditRequestUtils from litellm.secret_managers.main import get_secret_str @@ -220,11 +222,18 @@ class OpenAIVideoConfig(BaseVideoConfig): - GET /v1/videos/{video_id}/content?variant=thumbnail """ original_video_id = extract_original_video_id(video_id) + encoded_video_id = encode_url_path_segment( + original_video_id, field_name="video_id" + ) # Construct the URL for video content download - url = f"{api_base.rstrip('/')}/{original_video_id}/content" + url = f"{api_base.rstrip('/')}/{encoded_video_id}/content" if variant is not None: - url = f"{url}?variant={variant}" + # Encode the user-controlled ``variant`` so a value like + # ``thumbnail&extra=1`` cannot inject additional query params + # into the upstream request — same hardening rationale as the + # path-segment encoding above. + url = f"{url}?variant={quote(variant, safe='')}" # No additional data needed for GET content request data: Dict[str, Any] = {} @@ -247,9 +256,12 @@ class OpenAIVideoConfig(BaseVideoConfig): - POST /v1/videos/{video_id}/remix """ original_video_id = extract_original_video_id(video_id) + encoded_video_id = encode_url_path_segment( + original_video_id, field_name="video_id" + ) # Construct the URL for video remix - url = f"{api_base.rstrip('/')}/{original_video_id}/remix" + url = f"{api_base.rstrip('/')}/{encoded_video_id}/remix" # Prepare the request data data = {"prompt": prompt} @@ -391,9 +403,12 @@ class OpenAIVideoConfig(BaseVideoConfig): - DELETE /v1/videos/{video_id} """ original_video_id = extract_original_video_id(video_id) + encoded_video_id = encode_url_path_segment( + original_video_id, field_name="video_id" + ) # Construct the URL for video delete - url = f"{api_base.rstrip('/')}/{original_video_id}" + url = f"{api_base.rstrip('/')}/{encoded_video_id}" # No data needed for DELETE request data: Dict[str, Any] = {} @@ -427,9 +442,12 @@ class OpenAIVideoConfig(BaseVideoConfig): """ # Extract the original video_id (remove provider encoding if present) original_video_id = extract_original_video_id(video_id) + encoded_video_id = encode_url_path_segment( + original_video_id, field_name="video_id" + ) # For video retrieve, we just need to construct the URL - url = f"{api_base.rstrip('/')}/{original_video_id}" + url = f"{api_base.rstrip('/')}/{encoded_video_id}" # No additional data needed for GET request data: Dict[str, Any] = {} @@ -494,7 +512,11 @@ class OpenAIVideoConfig(BaseVideoConfig): litellm_params: GenericLiteLLMParams, headers: dict, ) -> Tuple[str, Dict]: - url = f"{api_base.rstrip('/')}/characters/{character_id}" + original_character_id = extract_original_character_id(character_id) + encoded_character_id = encode_url_path_segment( + original_character_id, field_name="character_id" + ) + url = f"{api_base.rstrip('/')}/characters/{encoded_character_id}" return url, {} def transform_video_get_character_response( diff --git a/litellm/llms/pg_vector/vector_stores/transformation.py b/litellm/llms/pg_vector/vector_stores/transformation.py index 7b22edd8676..fc4cfc7b083 100644 --- a/litellm/llms/pg_vector/vector_stores/transformation.py +++ b/litellm/llms/pg_vector/vector_stores/transformation.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams @@ -82,7 +83,10 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: - url = f"{api_base}/{vector_store_id}/search" + encoded_vector_store_id = encode_url_path_segment( + vector_store_id, field_name="vector_store_id" + ) + url = f"{api_base}/{encoded_vector_store_id}/search" _, request_body = super().transform_search_vector_store_request( vector_store_id=vector_store_id, query=query, diff --git a/litellm/llms/ragflow/chat/transformation.py b/litellm/llms/ragflow/chat/transformation.py index d49a5fd370f..990fc2b2e61 100644 --- a/litellm/llms/ragflow/chat/transformation.py +++ b/litellm/llms/ragflow/chat/transformation.py @@ -13,6 +13,7 @@ Model name format: from typing import List, Optional, Tuple import litellm +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.openai.openai import OpenAIConfig from litellm.secret_managers.main import get_secret, get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -126,10 +127,11 @@ class RAGFlowConfig(OpenAIConfig): api_base = api_base[:-3] # Remove /v1 # Construct the RAGFlow-specific path + encoded_entity_id = encode_url_path_segment(entity_id, field_name="entity_id") if endpoint_type == "chat": - path = f"/api/v1/chats_openai/{entity_id}/chat/completions" + path = f"/api/v1/chats_openai/{encoded_entity_id}/chat/completions" else: # agent - path = f"/api/v1/agents_openai/{entity_id}/chat/completions" + path = f"/api/v1/agents_openai/{encoded_entity_id}/chat/completions" # Ensure path starts with / if not path.startswith("/"): diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 8377dea952e..4f84816a2bc 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -6,6 +6,7 @@ from httpx._types import RequestFiles import litellm from litellm.constants import RUNWAYML_DEFAULT_API_VERSION +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.custom_httpx.http_handler import ( @@ -334,9 +335,12 @@ class RunwayMLVideoConfig(BaseVideoConfig): We'll retrieve the task and extract the video URL. """ original_video_id = extract_original_video_id(video_id) + encoded_video_id = encode_url_path_segment( + original_video_id, field_name="video_id" + ) # Get task status to retrieve video URL - url = f"{api_base}/tasks/{original_video_id}" + url = f"{api_base}/tasks/{encoded_video_id}" params: Dict[str, Any] = {} @@ -495,9 +499,12 @@ class RunwayMLVideoConfig(BaseVideoConfig): RunwayML uses task cancellation. """ original_video_id = extract_original_video_id(video_id) + encoded_video_id = encode_url_path_segment( + original_video_id, field_name="video_id" + ) # Construct the URL for task cancellation - url = f"{api_base}/tasks/{original_video_id}/cancel" + url = f"{api_base}/tasks/{encoded_video_id}/cancel" data: Dict[str, Any] = {} @@ -533,9 +540,12 @@ class RunwayMLVideoConfig(BaseVideoConfig): RunwayML uses GET /v1/tasks/{task_id} to retrieve task status. """ original_video_id = extract_original_video_id(video_id) + encoded_video_id = encode_url_path_segment( + original_video_id, field_name="video_id" + ) # Construct the full URL for task status retrieval - url = f"{api_base}/tasks/{original_video_id}" + url = f"{api_base}/tasks/{encoded_video_id}" # Empty dict for GET request (no body) data: Dict[str, Any] = {} diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index 7436bfef58b..c627599da8d 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -4,7 +4,11 @@ from typing import Any, Coroutine, Dict, Optional, Union import httpx import litellm -from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get +from litellm.litellm_core_utils.url_utils import ( + async_safe_get, + encode_url_path_segment, + safe_get, +) from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, @@ -170,7 +174,8 @@ class VertexAIBatchPrediction(VertexLLM): ) # Append batch_id to the URL - default_api_base = f"{default_api_base}/{batch_id}" + encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id") + default_api_base = f"{default_api_base}/{encoded_batch_id}" if len(default_api_base.split(":")) > 1: endpoint = default_api_base.split(":")[-1] @@ -413,7 +418,8 @@ class VertexAIBatchPrediction(VertexLLM): vertex_project=vertex_project or project_id, ) - retrieve_api_base_default = f"{default_api_base}/{batch_id}" + encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id") + retrieve_api_base_default = f"{default_api_base}/{encoded_batch_id}" cancel_api_base_default = f"{retrieve_api_base_default}:cancel" _, api_base = self._check_custom_proxy( diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index b4bfde5f541..c72160f7d0a 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -27,6 +27,53 @@ class VertexAIError(BaseLLMException): super().__init__(message=message, status_code=status_code, headers=headers) +def vertex_request_labels_from_litellm_params( + litellm_params: Optional[dict], +) -> Optional[Dict[str, str]]: + """ + Build Vertex/GCP billing labels from LiteLLM user metadata on ``litellm_params``: + ``metadata`` (``completion(..., metadata=...)``) or ``litellm_metadata``, + using ``requester_metadata`` string key-value pairs (same convention as Gemini). + ``metadata`` is tried first when both are present. + """ + if not litellm_params: + return None + for key in ("metadata", "litellm_metadata"): + if key not in litellm_params: + continue + metadata = litellm_params[key] + if metadata is None or not isinstance(metadata, dict): + continue + if "requester_metadata" not in metadata: + continue + rm = metadata["requester_metadata"] + if not isinstance(rm, dict): + continue + labels = {k: v for k, v in rm.items() if isinstance(v, str)} + if labels: + return labels + return None + + +def pop_vertex_request_labels( + optional_params: Optional[dict], + litellm_params: Optional[dict], +) -> Optional[Dict[str, str]]: + """ + Resolve labels from optional ``labels`` (Gemini-style) and/or + ``litellm_params["metadata"]`` / ``litellm_params["litellm_metadata"]`` + (``requester_metadata``). Pops ``labels`` from optional_params when present. + """ + labels: Optional[Dict[str, str]] = None + if optional_params is not None and "labels" in optional_params: + raw = optional_params.pop("labels") + if isinstance(raw, dict): + labels = {k: v for k, v in raw.items() if isinstance(v, str)} + if not labels: + labels = vertex_request_labels_from_litellm_params(litellm_params) + return labels if labels else None + + class VertexAIModelRoute(str, Enum): """Enum for Vertex AI model routing""" @@ -50,7 +97,7 @@ def get_vertex_ai_model_route( Determine which handler to use for a Vertex AI model based on the model name. Args: - model: The model name (e.g., "llama3-405b", "gemini-pro", "gemma/gemma-3-12b-it", "openai/gpt-oss-120b") + model: The model name (e.g., "llama3-405b", "gemini-pro", "gemma/gemma-3-12b-it", "xai/grok-4.1-fast-non-reasoning") litellm_params: Optional litellm parameters dict that may contain base_model for routing Returns: @@ -66,7 +113,7 @@ def get_vertex_ai_model_route( >>> get_vertex_ai_model_route("gemma/gemma-3-12b-it") VertexAIModelRoute.GEMMA - >>> get_vertex_ai_model_route("openai/gpt-oss-120b") + >>> get_vertex_ai_model_route("xai/grok-4.1-fast-non-reasoning") VertexAIModelRoute.MODEL_GARDEN >>> get_vertex_ai_model_route("1234567890", {"api_base": "http://10.96.32.8"}) @@ -102,8 +149,11 @@ def get_vertex_ai_model_route( if "gemma/" in model: return VertexAIModelRoute.GEMMA - # Check for model garden openai models - if "openai" in model: + # Check for model garden OpenAI-compatible publisher models. + # Examples: + # - openai/gpt-oss-120b-maas + # - xai/grok-4.1-fast-non-reasoning + if "openai" in model or model.startswith("xai/"): return VertexAIModelRoute.MODEL_GARDEN # Check for gemini models @@ -209,8 +259,8 @@ def get_vertex_base_model_name(model: str) -> str: >>> get_vertex_base_model_name("gemma/gemma-3-12b-it") "gemma-3-12b-it" - >>> get_vertex_base_model_name("openai/gpt-oss-120b") - "gpt-oss-120b" + >>> get_vertex_base_model_name("xai/grok-4.1-fast-non-reasoning") + "grok-4.1-fast-non-reasoning" >>> get_vertex_base_model_name("1234567890") "1234567890" diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 533bd06d2d8..7fa220a59fa 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( response_schema_prompt, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels from litellm.types.files import ( get_file_mime_type_for_file_type, get_file_type_from_extension, @@ -714,16 +715,8 @@ def _transform_request_body( # noqa: PLR0915 optional_params.pop("output_config", None) config_fields = GenerationConfig.__annotations__.keys() - # If the LiteLLM client sends Gemini-supported parameter "labels", add it - # as "labels" field to the request sent to the Gemini backend. - labels: Optional[dict[str, str]] = optional_params.pop("labels", None) - # If the LiteLLM client sends OpenAI-supported parameter "metadata", add it - # as "labels" field to the request sent to the Gemini backend. - if labels is None and "metadata" in litellm_params: - metadata = litellm_params["metadata"] - if metadata is not None and "requester_metadata" in metadata: - rm = metadata["requester_metadata"] - labels = {k: v for k, v in rm.items() if isinstance(v, str)} + # labels: optional explicit param and/or metadata.requester_metadata (OpenAI metadata) + labels = pop_vertex_request_labels(optional_params, litellm_params) filtered_params = { k: v @@ -750,16 +743,22 @@ def _transform_request_body( # noqa: PLR0915 ] data = RequestBody(contents=content) - if system_instructions is not None: - data["system_instruction"] = system_instructions - if tools is not None: - data["tools"] = tools - if tool_choice is not None: - data["toolConfig"] = tool_choice - if include_server_side_tool_invocations: - if "toolConfig" not in data: - data["toolConfig"] = {} - data["toolConfig"]["includeServerSideToolInvocations"] = True + # Vertex rejects system_instruction/tools/toolConfig alongside cachedContent. + # Treat dropping these fields as a request mutation guarded by modify_params. + can_send_cache_incompatible_fields = ( + cached_content is None or litellm.modify_params is False + ) + if can_send_cache_incompatible_fields: + if system_instructions is not None: + data["system_instruction"] = system_instructions + if tools is not None: + data["tools"] = tools + if tool_choice is not None: + data["toolConfig"] = tool_choice + if include_server_side_tool_invocations: + if "toolConfig" not in data: + data["toolConfig"] = {} + data["toolConfig"]["includeServerSideToolInvocations"] = True if safety_settings is not None: data["safetySettings"] = safety_settings if generation_config is not None and len(generation_config) > 0: diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py index 2371bc4865a..99165c37c93 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py @@ -3,7 +3,7 @@ Google AI Studio /batchEmbedContents Embeddings Endpoint """ import json -from typing import Any, Dict, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Tuple, Union import httpx @@ -13,8 +13,8 @@ from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, get_async_httpx_client, ) -from litellm.types.llms.openai import EmbeddingInput from litellm.types.llms.vertex_ai import ( + GeminiEmbeddingInput, VertexAIBatchEmbeddingsRequestBody, VertexAIBatchEmbeddingsResponseObject, ) @@ -23,7 +23,6 @@ from litellm.types.utils import EmbeddingResponse from ..gemini.vertex_and_google_ai_studio_gemini import VertexLLM from .batch_embed_content_transformation import ( _is_file_reference, - _is_multimodal_input, process_embed_content_response, process_response, transform_openai_input_gemini_content, @@ -32,9 +31,24 @@ from .batch_embed_content_transformation import ( class GoogleBatchEmbeddings(VertexLLM): + @staticmethod + def _flatten_and_detect_file_refs( + input: GeminiEmbeddingInput, + ) -> Tuple[List[str], bool]: + """Flatten nested input lists and detect file references.""" + input_list = [input] if isinstance(input, str) else input + flat_elements = [ + e + for item in input_list + for e in (item if isinstance(item, list) else [item]) + if isinstance(e, str) + ] + has_file_refs = any(_is_file_reference(e) for e in flat_elements) + return flat_elements, has_file_refs + def _resolve_file_references( self, - input: EmbeddingInput, + input: GeminiEmbeddingInput, api_key: str, sync_handler: HTTPHandler, ) -> Dict[str, Dict[str, str]]: @@ -42,7 +56,7 @@ class GoogleBatchEmbeddings(VertexLLM): Resolve Gemini file references (files/...) to get mime_type and uri. Args: - input: EmbeddingInput that may contain file references + input: GeminiEmbeddingInput that may contain file references api_key: Gemini API key sync_handler: HTTP client @@ -73,7 +87,7 @@ class GoogleBatchEmbeddings(VertexLLM): async def _async_resolve_file_references( self, - input: EmbeddingInput, + input: GeminiEmbeddingInput, api_key: str, async_handler: AsyncHTTPHandler, ) -> Dict[str, Dict[str, str]]: @@ -81,7 +95,7 @@ class GoogleBatchEmbeddings(VertexLLM): Async version of _resolve_file_references. Args: - input: EmbeddingInput that may contain file references + input: GeminiEmbeddingInput that may contain file references api_key: Gemini API key async_handler: Async HTTP client @@ -110,10 +124,10 @@ class GoogleBatchEmbeddings(VertexLLM): return resolved_files - def batch_embeddings( + def batch_embeddings( # noqa: PLR0915 self, model: str, - input: EmbeddingInput, + input: GeminiEmbeddingInput, print_verbose, model_response: EmbeddingResponse, custom_llm_provider: Literal["gemini", "vertex_ai"], @@ -151,8 +165,7 @@ class GoogleBatchEmbeddings(VertexLLM): optional_params = optional_params or {} - is_multimodal = _is_multimodal_input(input) - use_embed_content = is_multimodal or (custom_llm_provider == "vertex_ai") + use_embed_content = custom_llm_provider == "vertex_ai" mode: Literal["embedding", "batch_embedding"] if use_embed_content: mode = "embedding" @@ -215,8 +228,22 @@ class GoogleBatchEmbeddings(VertexLLM): resolved_files=resolved_files, ) else: + flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input) + if has_file_refs and not api_key: + raise ValueError( + "An API key is required to resolve Gemini file references (files/...). " + "Pass api_key= or set GEMINI_API_KEY." + ) + resolved_files = {} + if api_key and has_file_refs: + resolved_files = self._resolve_file_references( + input=flat_elements, api_key=api_key, sync_handler=sync_handler + ) request_data = transform_openai_input_gemini_content( - input=input, model=model, optional_params=optional_params + input=input, + model=model, + optional_params=optional_params, + resolved_files=resolved_files, ) ## LOGGING @@ -264,7 +291,7 @@ class GoogleBatchEmbeddings(VertexLLM): url: str, data: Optional[Union[VertexAIBatchEmbeddingsRequestBody, dict]], model_response: EmbeddingResponse, - input: EmbeddingInput, + input: GeminiEmbeddingInput, timeout: Optional[Union[float, httpx.Timeout]], headers={}, client: Optional[AsyncHTTPHandler] = None, @@ -303,8 +330,22 @@ class GoogleBatchEmbeddings(VertexLLM): resolved_files=resolved_files, ) else: + flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input) + if has_file_refs and not api_key: + raise ValueError( + "An API key is required to resolve Gemini file references (files/...). " + "Pass api_key= or set GEMINI_API_KEY." + ) + resolved_files = {} + if api_key and has_file_refs: + resolved_files = await self._async_resolve_file_references( + input=flat_elements, api_key=api_key, async_handler=async_handler + ) data = transform_openai_input_gemini_content( - input=input, model=model, optional_params=optional_params or {} + input=input, + model=model, + optional_params=optional_params or {}, + resolved_files=resolved_files, ) ## LOGGING diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 34fc95e0af7..e1b365c9f42 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -6,12 +6,12 @@ Why separate file? Make it easy to see how transformation works from typing import Dict, List, Optional, Tuple -from litellm.types.llms.openai import EmbeddingInput from litellm.types.llms.vertex_ai import ( BlobType, ContentType, EmbedContentRequest, FileDataType, + GeminiEmbeddingInput, PartType, VertexAIBatchEmbeddingsRequestBody, VertexAIBatchEmbeddingsResponseObject, @@ -114,33 +114,77 @@ def _parse_data_url(data_url: str) -> Tuple[str, str]: return media_type, base64_data -def _is_multimodal_input(input: EmbeddingInput) -> bool: +def _is_multimodal_input(input: GeminiEmbeddingInput) -> bool: """ - Check if the input contains multimodal data (data URIs, file references, or GCS URLs). + Check if the input contains multimodal data (data URIs, file references, + GCS URLs, or nested lists for combined embeddings). Args: - input: EmbeddingInput (str or List[str]) + input: GeminiEmbeddingInput — str, List[str], or List[List[str]] for combined embeddings Returns: - bool: True if any element is a data URI, file reference, or GCS URL + bool: True if any element is multimodal or a nested list """ if isinstance(input, str): - input_list = [input] - else: - input_list = input + return _is_multimodal_element(input) - for element in input_list: - if isinstance(element, str): - if element.startswith("data:") and ";base64," in element: - return True - if _is_file_reference(element): - return True - if _is_gcs_url(element): + for element in input: + if isinstance(element, list): + if any( + _is_multimodal_element(sub) for sub in element if isinstance(sub, str) + ): return True + elif isinstance(element, str) and _is_multimodal_element(element): + return True return False +def _is_multimodal_element(element: str) -> bool: + """Check if a single string element is multimodal.""" + if element.startswith("data:") and ";base64," in element: + return True + if _is_file_reference(element): + return True + if _is_gcs_url(element): + return True + return False + + +def _build_part_for_input( + element: str, + resolved_files: Optional[Dict[str, Dict[str, str]]] = None, +) -> PartType: + """ + Build a single PartType for an input element, handling text, data URIs, + file references, and GCS URLs. + """ + resolved_files = resolved_files or {} + + if element.startswith("data:") and ";base64," in element: + mime_type, base64_data = _parse_data_url(element) + blob: BlobType = {"mime_type": mime_type, "data": base64_data} + return PartType(inline_data=blob) + elif _is_gcs_url(element): + mime_type = _infer_mime_type_from_gcs_url(element) + file_data: FileDataType = { + "mime_type": mime_type, + "file_uri": element, + } + return PartType(file_data=file_data) + elif _is_file_reference(element): + if element not in resolved_files: + raise ValueError(f"File reference {element} not resolved") + file_info = resolved_files[element] + file_data_ref: FileDataType = { + "mime_type": file_info["mime_type"], + "file_uri": file_info["uri"], + } + return PartType(file_data=file_data_ref) + else: + return PartType(text=element) + + _SUPPORTED_EMBED_PARAMS = {"outputDimensionality", "taskType", "title"} @@ -155,37 +199,60 @@ def _filter_embed_params(optional_params: dict) -> dict: def transform_openai_input_gemini_content( - input: EmbeddingInput, model: str, optional_params: dict + input: GeminiEmbeddingInput, + model: str, + optional_params: dict, + resolved_files: Optional[Dict[str, Dict[str, str]]] = None, ) -> VertexAIBatchEmbeddingsRequestBody: """ - The content to embed. Only the parts.text fields will be counted. + Transform OpenAI embedding input to Gemini batchEmbedContents format. + + Each input element becomes a separate EmbedContentRequest, supporting + text, data URIs, file references, and GCS URLs. + + If an element is a list (nested input), all sub-elements are combined + into a single content with multiple parts, producing one combined + embedding for the group. + + Examples: + input=["text", "image"] → 2 separate embeddings + input=[["text", "image"]] → 1 combined embedding + input=[["text", "image"], "x"] → 2 embeddings (1 combined + 1 separate) """ gemini_model_name = "models/{}".format(model) gemini_params = _filter_embed_params(optional_params) + input_list = [input] if isinstance(input, str) else input requests: List[EmbedContentRequest] = [] - if isinstance(input, str): + + for element in input_list: + if isinstance(element, list): + if not element: + raise ValueError("Nested input list must not be empty") + for sub in element: + if not isinstance(sub, str): + raise ValueError( + f"Elements inside a nested input list must be strings, got {type(sub)}" + ) + parts = [ + _build_part_for_input(sub, resolved_files=resolved_files) + for sub in element + ] + else: + parts = [_build_part_for_input(element, resolved_files=resolved_files)] request = EmbedContentRequest( model=gemini_model_name, - content=ContentType(parts=[PartType(text=input)]), + content=ContentType(parts=parts), **gemini_params, ) requests.append(request) - else: - for i in input: - request = EmbedContentRequest( - model=gemini_model_name, - content=ContentType(parts=[PartType(text=i)]), - **gemini_params, - ) - requests.append(request) return VertexAIBatchEmbeddingsRequestBody(requests=requests) def transform_openai_input_gemini_embed_content( - input: EmbeddingInput, + input: GeminiEmbeddingInput, model: str, optional_params: dict, resolved_files: Optional[Dict[str, Dict[str, str]]] = None, @@ -194,7 +261,7 @@ def transform_openai_input_gemini_embed_content( Transform OpenAI embedding input to Gemini embedContent format (multimodal). Args: - input: EmbeddingInput (str or List[str]) with text, data URIs, or file references + input: GeminiEmbeddingInput with text, data URIs, or file references model: Model name optional_params: Additional parameters (taskType, outputDimensionality, etc.) resolved_files: Dict mapping file names (files/abc) to {mime_type, uri} @@ -210,31 +277,14 @@ def transform_openai_input_gemini_embed_content( parts: List[PartType] = [] for element in input_list: + if isinstance(element, list): + raise ValueError( + "Nested (combined) embeddings are not supported on the embedContent path. " + "Use the batchEmbedContents path or pass a flat list instead." + ) if not isinstance(element, str): raise ValueError(f"Unsupported input type: {type(element)}") - - if element.startswith("data:") and ";base64," in element: - mime_type, base64_data = _parse_data_url(element) - blob: BlobType = {"mime_type": mime_type, "data": base64_data} - parts.append(PartType(inline_data=blob)) - elif _is_gcs_url(element): - mime_type = _infer_mime_type_from_gcs_url(element) - file_data: FileDataType = { - "mime_type": mime_type, - "file_uri": element, - } - parts.append(PartType(file_data=file_data)) - elif _is_file_reference(element): - if element not in resolved_files: - raise ValueError(f"File reference {element} not resolved") - file_info = resolved_files[element] - file_data_ref: FileDataType = { - "mime_type": file_info["mime_type"], - "file_uri": file_info["uri"], - } - parts.append(PartType(file_data=file_data_ref)) - else: - parts.append(PartType(text=element)) + parts.append(_build_part_for_input(element, resolved_files=resolved_files)) request_body: dict = { "content": ContentType(parts=parts), @@ -245,7 +295,7 @@ def transform_openai_input_gemini_embed_content( def process_embed_content_response( - input: EmbeddingInput, + input: GeminiEmbeddingInput, model_response: EmbeddingResponse, model: str, response_json: dict, @@ -291,7 +341,7 @@ def process_embed_content_response( def process_response( - input: EmbeddingInput, + input: GeminiEmbeddingInput, model_response: EmbeddingResponse, model: str, _predictions: VertexAIBatchEmbeddingsResponseObject, @@ -308,8 +358,29 @@ def process_response( model_response.data = openai_embeddings model_response.model = model - input_text = get_formatted_prompt(data={"input": input}, call_type="embedding") - prompt_tokens = token_counter(model=model, text=input_text) + has_nested = isinstance(input, list) and any(isinstance(e, list) for e in input) + if _is_multimodal_input(input) or has_nested: + input_list = input if isinstance(input, list) else [input] + text_elements: List[str] = [] + for e in input_list: + if isinstance(e, list): + text_elements.extend( + sub + for sub in e + if isinstance(sub, str) and not _is_multimodal_element(sub) + ) + elif isinstance(e, str) and not _is_multimodal_element(e): + text_elements.append(e) + if text_elements: + input_text = get_formatted_prompt( + data={"input": text_elements}, call_type="embedding" + ) + prompt_tokens = token_counter(model=model, text=input_text) + else: + prompt_tokens = 0 + else: + input_text = get_formatted_prompt(data={"input": input}, call_type="embedding") + prompt_tokens = token_counter(model=model, text=input_text) model_response.usage = Usage( prompt_tokens=prompt_tokens, total_tokens=prompt_tokens ) diff --git a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py index 1c7696d55a2..05ebd685d91 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py @@ -7,7 +7,10 @@ import litellm from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) -from litellm.llms.vertex_ai.common_utils import get_vertex_base_url +from litellm.llms.vertex_ai.common_utils import ( + get_vertex_base_url, + pop_vertex_request_labels, +) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -203,13 +206,16 @@ class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): "sampleCount": 1, } - # Merge with optional params + labels = pop_vertex_request_labels(optional_params, litellm_params) + # Merge with optional params (after popping labels so they are not sent as Imagen parameters) parameters = {**default_params, **optional_params} - request_body = { + request_body: dict = { "instances": [{"prompt": prompt}], "parameters": parameters, } + if labels: + request_body["labels"] = labels return request_body diff --git a/litellm/llms/vertex_ai/rerank/transformation.py b/litellm/llms/vertex_ai/rerank/transformation.py index 53651839671..3b84972e946 100644 --- a/litellm/llms/vertex_ai/rerank/transformation.py +++ b/litellm/llms/vertex_ai/rerank/transformation.py @@ -11,12 +11,15 @@ import httpx import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig +from litellm.llms.vertex_ai.common_utils import ( + vertex_request_labels_from_litellm_params, +) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.secret_managers.main import get_secret_str from litellm.types.rerank import ( + RerankBilledUnits, RerankResponse, RerankResponseMeta, - RerankBilledUnits, RerankResponseResult, ) @@ -109,6 +112,7 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: """ Transform the request from Cohere format to Vertex AI Discovery Engine format @@ -145,6 +149,10 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase): # When return_documents is False, we want to ignore record details (return only IDs) request_data["ignoreRecordDetailsInResponse"] = not return_documents + user_labels = vertex_request_labels_from_litellm_params(litellm_params) + if user_labels: + request_data["userLabels"] = user_labels + return request_data def transform_rerank_response( diff --git a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py index 6cb7a86bea2..61fb848b40a 100644 --- a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import httpx from litellm import get_model_info +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.types.router import GenericLiteLLMParams @@ -91,12 +92,18 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): raise ValueError("vector_store_id is required") if api_base: return api_base.rstrip("/") + encoded_collection_id = encode_url_path_segment( + collection_id, field_name="vertex_collection_id" + ) + encoded_datastore_id = encode_url_path_segment( + datastore_id, field_name="vector_store_id" + ) # Vertex AI Search API endpoint for search return ( f"https://discoveryengine.googleapis.com/v1/" f"projects/{vertex_project}/locations/{vertex_location}/" - f"collections/{collection_id}/dataStores/{datastore_id}/servingConfigs/default_config" + f"collections/{encoded_collection_id}/dataStores/{encoded_datastore_id}/servingConfigs/default_config" ) def transform_search_vector_store_request( diff --git a/litellm/llms/vertex_ai/vertex_embeddings/embedding_handler.py b/litellm/llms/vertex_ai/vertex_embeddings/embedding_handler.py index 18c5ec3d839..696341598e5 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/embedding_handler.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/embedding_handler.py @@ -1,4 +1,4 @@ -from typing import Literal, Optional, Union +from typing import Dict, Literal, Optional, Union import httpx @@ -44,6 +44,7 @@ class VertexEmbedding(VertexBase): vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES] = None, gemini_api_key: Optional[str] = None, extra_headers: Optional[dict] = None, + litellm_params: Optional[Dict] = None, ) -> EmbeddingResponse: if aembedding is True: return self.async_embedding( # type: ignore @@ -61,6 +62,7 @@ class VertexEmbedding(VertexBase): vertex_credentials=vertex_credentials, gemini_api_key=gemini_api_key, extra_headers=extra_headers, + litellm_params=litellm_params, ) should_use_v1beta1_features = self.is_using_v1beta1_features( @@ -92,7 +94,10 @@ class VertexEmbedding(VertexBase): headers = self.set_headers(auth_header=auth_header, extra_headers=extra_headers) vertex_request: VertexEmbeddingRequest = ( litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request( - input=input, optional_params=optional_params, model=model + input=input, + optional_params=optional_params, + model=model, + litellm_params=litellm_params, ) ) @@ -156,6 +161,7 @@ class VertexEmbedding(VertexBase): gemini_api_key: Optional[str] = None, extra_headers: Optional[dict] = None, encoding=None, + litellm_params: Optional[Dict] = None, ) -> EmbeddingResponse: """ Async embedding implementation @@ -188,7 +194,10 @@ class VertexEmbedding(VertexBase): headers = self.set_headers(auth_header=auth_header, extra_headers=extra_headers) vertex_request: VertexEmbeddingRequest = ( litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request( - input=input, optional_params=optional_params, model=model + input=input, + optional_params=optional_params, + model=model, + litellm_params=litellm_params, ) ) diff --git a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py index 132f29987af..24396628dbd 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py @@ -3,6 +3,7 @@ from typing import List, Literal, Optional, Union from pydantic import BaseModel +from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels from litellm.types.utils import EmbeddingResponse, Usage from .types import * @@ -100,7 +101,11 @@ class VertexAITextEmbeddingConfig(BaseModel): return optional_params def transform_openai_request_to_vertex_embedding_request( - self, input: Union[list, str], optional_params: dict, model: str + self, + input: Union[list, str], + optional_params: dict, + model: str, + litellm_params: Optional[dict] = None, ) -> VertexEmbeddingRequest: """ Transforms an openai request to a vertex embedding request. @@ -108,16 +113,26 @@ class VertexAITextEmbeddingConfig(BaseModel): # Import here to avoid circular import issues with litellm.__init__ from litellm.llms.vertex_ai.vertex_embeddings.bge import VertexBGEConfig + labels = pop_vertex_request_labels(optional_params, litellm_params) + if model.isdigit(): - return self._transform_openai_request_to_fine_tuned_embedding_request( - input, optional_params, model + vertex_request = ( + self._transform_openai_request_to_fine_tuned_embedding_request( + input, optional_params, model + ) ) + if labels: + vertex_request["labels"] = labels + return vertex_request if VertexBGEConfig.is_bge_model(model): - return VertexBGEConfig.transform_request( + vertex_request = VertexBGEConfig.transform_request( input=input, optional_params=optional_params, model=model ) + if labels: + vertex_request["labels"] = labels + return vertex_request - vertex_request: VertexEmbeddingRequest = VertexEmbeddingRequest() + vertex_request = VertexEmbeddingRequest() vertex_text_embedding_input_list: List[TextEmbeddingInput] = [] task_type: Optional[TaskType] = optional_params.get("task_type") title = optional_params.get("title") @@ -133,6 +148,8 @@ class VertexAITextEmbeddingConfig(BaseModel): vertex_request["instances"] = vertex_text_embedding_input_list vertex_request["parameters"] = EmbeddingParameters(**optional_params) + if labels: + vertex_request["labels"] = labels return vertex_request diff --git a/litellm/llms/vertex_ai/vertex_embeddings/types.py b/litellm/llms/vertex_ai/vertex_embeddings/types.py index 317b9c4fb81..bf73f4d193a 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/types.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/types.py @@ -3,7 +3,7 @@ Types for Vertex Embeddings Requests """ from enum import Enum -from typing import List, Optional, Union +from typing import Dict, List, Optional, Union from typing_extensions import TypedDict @@ -56,6 +56,7 @@ class VertexEmbeddingRequest(TypedDict, total=False): List[TextEmbeddingFineTunedInput], ] parameters: Optional[Union[EmbeddingParameters, TextEmbeddingFineTunedParameters]] + labels: Optional[Dict[str, str]] # Example usage: diff --git a/litellm/llms/vertex_ai/vertex_model_garden/main.py b/litellm/llms/vertex_ai/vertex_model_garden/main.py index c37bb449ecf..7240d9dce57 100644 --- a/litellm/llms/vertex_ai/vertex_model_garden/main.py +++ b/litellm/llms/vertex_ai/vertex_model_garden/main.py @@ -27,6 +27,17 @@ from ..common_utils import VertexAIError, get_vertex_base_model_name from ..vertex_llm_base import VertexBase +def _vertex_model_garden_model_id_in_json_body(model: str) -> bool: + """ + Vertex catalog / publisher models are addressed as publisher/model (e.g. + xai/grok-4.1-fast-reasoning) on the shared OpenAPI URL, with the id in the JSON body. + + Deployed Model Garden endpoints are typically a single segment (often numeric) + and use .../endpoints/{ENDPOINT_ID}/chat/completions with an empty model field. + """ + return "/" in model + + def create_vertex_url( vertex_location: str, vertex_project: str, @@ -34,8 +45,13 @@ def create_vertex_url( model: str, api_base: Optional[str] = None, ) -> str: - """Return the base url for the vertex garden models""" + """Return the api base for vertex model garden (without /chat/completions).""" base_url = get_vertex_base_url(vertex_location) + if _vertex_model_garden_model_id_in_json_body(model): + return ( + f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}" + "/endpoints/openapi" + ) return f"{base_url}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}" @@ -129,7 +145,10 @@ class VertexAIModelGardenModels(VertexBase): vertex_location=vertex_location or "us-central1", vertex_api_version="v1beta1", ) - model = "" + # Publisher/catalog models: model id must be sent in the JSON body (OpenAPI route). + # Single-segment endpoint ids: model is encoded in the URL path; body model stays empty. + if not _vertex_model_garden_model_id_in_json_body(model): + model = "" return openai_like_chat_completions.completion( model=model, messages=messages, diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index f6dda4dd25b..99e0a958ef1 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -17,6 +17,7 @@ from pydantic import fields as pyd_fields import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( _safe_convert_created_field, ) @@ -300,7 +301,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): litellm_params: GenericLiteLLMParams, headers: dict, ) -> Tuple[str, Dict]: - url = f"{api_base}/{response_id}" + encoded_response_id = encode_url_path_segment( + response_id, field_name="response_id" + ) + url = f"{api_base}/{encoded_response_id}" data: Dict = {} return url, data @@ -333,7 +337,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): litellm_params: GenericLiteLLMParams, headers: dict, ) -> Tuple[str, Dict]: - url = f"{api_base}/{response_id}" + encoded_response_id = encode_url_path_segment( + response_id, field_name="response_id" + ) + url = f"{api_base}/{encoded_response_id}" data: Dict = {} return url, data @@ -372,7 +379,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): limit: int = 20, order: Literal["asc", "desc"] = "desc", ) -> Tuple[str, Dict]: - url = f"{api_base}/{response_id}/input_items" + encoded_response_id = encode_url_path_segment( + response_id, field_name="response_id" + ) + url = f"{api_base}/{encoded_response_id}/input_items" params: Dict[str, Any] = {} if after is not None: params["after"] = after @@ -408,7 +418,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): litellm_params: GenericLiteLLMParams, headers: dict, ) -> Tuple[str, Dict]: - url = f"{api_base}/{response_id}/cancel" + encoded_response_id = encode_url_path_segment( + response_id, field_name="response_id" + ) + url = f"{api_base}/{encoded_response_id}/cancel" data: Dict = {} return url, data diff --git a/litellm/llms/voyage/rerank/transformation.py b/litellm/llms/voyage/rerank/transformation.py index 521dae980d5..d64450a1211 100644 --- a/litellm/llms/voyage/rerank/transformation.py +++ b/litellm/llms/voyage/rerank/transformation.py @@ -67,7 +67,11 @@ class VoyageRerankConfig(BaseRerankConfig): return api_base def transform_rerank_request( - self, model: str, optional_rerank_params: Dict, headers: Dict + self, + model: str, + optional_rerank_params: Dict, + headers: Dict, + litellm_params: Optional[dict] = None, ) -> Dict: return {"model": model, **optional_rerank_params} diff --git a/litellm/llms/watsonx/rerank/transformation.py b/litellm/llms/watsonx/rerank/transformation.py index 4f8e196f254..202760f68a6 100644 --- a/litellm/llms/watsonx/rerank/transformation.py +++ b/litellm/llms/watsonx/rerank/transformation.py @@ -143,6 +143,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): model: str, optional_rerank_params: Dict, headers: dict, + litellm_params: Optional[dict] = None, ) -> dict: """ Transform request to IBM watsonx.ai rerank format diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index bfa55105a6c..64b4a545acb 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -43,6 +43,7 @@ class XAIChatConfig(OpenAIGPTConfig): "logprobs", "max_tokens", "n", + "parallel_tool_calls", "presence_penalty", "response_format", "seed", diff --git a/litellm/main.py b/litellm/main.py index daa0fb063aa..0553cf9d422 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4923,8 +4923,17 @@ def embedding( # noqa: PLR0915 if encoding_format is not None: optional_params["encoding_format"] = encoding_format else: - # Omiting causes openai sdk to add default value of "float" - optional_params["encoding_format"] = None + env_fmt = get_secret_str("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT") + if env_fmt is not None and env_fmt.strip().lower() == "none": + optional_params.pop("encoding_format", None) + else: + _default_fmt = ( + optional_params.get("encoding_format") or env_fmt or "float" + ) + if _default_fmt.strip().lower() == "none": + optional_params.pop("encoding_format", None) + else: + optional_params["encoding_format"] = _default_fmt api_version = None @@ -5311,6 +5320,7 @@ def embedding( # noqa: PLR0915 api_key=api_key, api_base=api_base, client=client, + litellm_params=litellm_params_dict, ) elif custom_llm_provider == "oobabooga": response = oobabooga.embedding( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e4268fac81a..a1e3e42a9c5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -712,6 +712,7 @@ }, "anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "bedrock_converse", @@ -735,6 +736,7 @@ }, "anthropic.claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "bedrock_converse", @@ -955,6 +957,7 @@ }, "anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", @@ -982,6 +985,7 @@ }, "anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", @@ -1011,6 +1015,7 @@ }, "global.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", @@ -1040,6 +1045,7 @@ }, "us.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1127,6 +1133,7 @@ }, "anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", @@ -1157,6 +1164,7 @@ }, "global.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", @@ -1187,6 +1195,7 @@ }, "us.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1277,6 +1286,7 @@ }, "anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "bedrock_converse", @@ -1305,6 +1315,7 @@ }, "global.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "bedrock_converse", @@ -1333,6 +1344,7 @@ }, "us.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1447,11 +1459,13 @@ }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -17921,11 +17935,13 @@ }, "global.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -17982,6 +17998,7 @@ }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "bedrock_converse", @@ -30116,6 +30133,7 @@ }, "us.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", @@ -30267,11 +30285,13 @@ }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -30372,6 +30392,7 @@ }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -30399,6 +30420,7 @@ }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "bedrock_converse", @@ -33315,6 +33337,72 @@ "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas", "supports_reasoning": true }, + "vertex_ai/xai/grok-4.1-fast-non-reasoning": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "vertex_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 2000000, + "max_tokens": 2000000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "vertex_ai/xai/grok-4.1-fast-reasoning": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "vertex_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 2000000, + "max_tokens": 2000000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "vertex_ai/xai/grok-4.20-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 2000000, + "max_tokens": 2000000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "vertex_ai/xai/grok-4.20-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 2000000, + "max_tokens": 2000000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas": { "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-qwen_models", diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index edee50bdfc4..c4c9aea6f64 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -21,6 +21,7 @@ import httpx from httpx._types import CookieTypes, QueryParamTypes, RequestFiles import litellm +from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler @@ -390,19 +391,28 @@ def _sync_streaming( ): from litellm.utils import executor + raw_bytes: List[bytes] = [] + flush_scheduled = False try: - raw_bytes: List[bytes] = [] for chunk in response.iter_bytes(): # type: ignore raw_bytes.append(chunk) yield chunk - - executor.submit( - litellm_logging_obj.flush_passthrough_collected_chunks, - raw_bytes=raw_bytes, - provider_config=provider_config, - ) - except Exception as e: - raise e + finally: + if not flush_scheduled and raw_bytes: + flush_scheduled = True + try: + executor.submit( + litellm_logging_obj.flush_passthrough_collected_chunks, + raw_bytes=raw_bytes, + provider_config=provider_config, + ) + except Exception as e: + verbose_logger.exception( + "Failed to schedule passthrough spend-tracking flush " + "in _sync_streaming; %d buffered chunks dropped: %s", + len(raw_bytes), + e, + ) async def _async_streaming( @@ -411,23 +421,45 @@ async def _async_streaming( provider_config: "BasePassthroughConfig", ): iter_response = await response + try: iter_response.raise_for_status() - raw_bytes: List[bytes] = [] - - async for chunk in iter_response.aiter_bytes(): # type: ignore - raw_bytes.append(chunk) - yield chunk - - asyncio.create_task( - litellm_logging_obj.async_flush_passthrough_collected_chunks( - raw_bytes=raw_bytes, - provider_config=provider_config, - ) - ) except Exception: try: await iter_response.aclose() except Exception: pass raise + + raw_bytes: List[bytes] = [] + flush_scheduled = False + try: + async for chunk in iter_response.aiter_bytes(): # type: ignore + raw_bytes.append(chunk) + yield chunk + except Exception: + try: + await iter_response.aclose() + except Exception: + pass + raise + finally: + # GeneratorExit (raised on client disconnect) is not caught by + # `except Exception`; the finally block ensures partial usage + # still gets flushed for spend tracking. See LIT-2642. + if not flush_scheduled and raw_bytes: + flush_scheduled = True + try: + asyncio.create_task( + litellm_logging_obj.async_flush_passthrough_collected_chunks( + raw_bytes=raw_bytes, + provider_config=provider_config, + ) + ) + except Exception as e: + verbose_logger.exception( + "Failed to schedule passthrough spend-tracking flush " + "in _async_streaming; %d buffered chunks dropped: %s", + len(raw_bytes), + e, + ) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index d4799e9f208..756b2ed91d7 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -117,7 +117,10 @@ class MCPRequestHandler: return b"{}" request.body = mock_body # type: ignore - if ".well-known" in str(request.url): # public routes + # Only OAuth metadata routes registered under /.well-known/ are public. + # Match on request.url.path (path-only, exact prefix) so the substring + # cannot be smuggled via query string, hostname, or a deeper URL segment. + if request.url.path.startswith("/.well-known/"): validated_user_api_key_auth = UserAPIKeyAuth() elif has_explicit_litellm_key: # Explicit x-litellm-api-key provided - always validate normally @@ -126,27 +129,37 @@ class MCPRequestHandler: ) elif oauth2_headers: # No x-litellm-api-key, but Authorization header present. - # Could be a LiteLLM key (backward compat) OR an OAuth2 token - # from an upstream MCP provider (e.g. Atlassian). - # Try LiteLLM auth first; on auth failure, treat as OAuth2 passthrough. + # Could be a LiteLLM key (backward compat) OR an opaque OAuth2 token + # the operator wants forwarded to an upstream OAuth2-mode MCP server. + # Try LiteLLM auth first; on auth failure, only fall back to anonymous + # passthrough when the request actually targets a server whose operator + # configured ``auth_type=oauth2``. For any other server (api_key, + # bearer_token, basic, etc.), a failed LiteLLM auth is a real failure + # and must propagate — otherwise an attacker can exchange any garbage + # bearer for an anonymous session. try: validated_user_api_key_auth = await user_api_key_auth( api_key=litellm_api_key, request=request ) - except HTTPException as e: - if e.status_code in (401, 403): + except (HTTPException, ProxyException) as e: + # HTTPException.status_code is int; ProxyException.code is + # normalized to str in its __init__ but can be ``"None"`` or any + # non-numeric string when the caller didn't supply a numeric + # code, so we compare against both int and str forms rather + # than coercing (``int("None")`` would raise ValueError and + # rewrite the auth error as a 500). + status = e.status_code if isinstance(e, HTTPException) else e.code + if status in ( + 401, + 403, + "401", + "403", + ) and MCPRequestHandler._target_servers_use_oauth2( + path=request.url.path, mcp_servers=mcp_servers + ): verbose_logger.debug( - "MCP OAuth2: Authorization header is not a valid LiteLLM key, " - "treating as OAuth2 token passthrough" - ) - validated_user_api_key_auth = UserAPIKeyAuth() - else: - raise - except ProxyException as e: - if str(e.code) in ("401", "403"): - verbose_logger.debug( - "MCP OAuth2: Authorization header is not a valid LiteLLM key, " - "treating as OAuth2 token passthrough" + "MCP OAuth2: target server is OAuth2-mode, treating " + "Authorization as upstream OAuth2 token passthrough" ) validated_user_api_key_auth = UserAPIKeyAuth() else: @@ -165,6 +178,62 @@ class MCPRequestHandler: dict(headers), ) + @staticmethod + def _extract_target_server_names_from_path(path: str) -> List[str]: + """ + Extract the target MCP server name from the standard MCP transport + URL patterns: ``/mcp/{server_name}[/...]`` and + ``/{server_name}/mcp[/...]``. Returns ``[]`` for any other path so + callers fail closed when the target cannot be resolved. + + REST/admin endpoints, OAuth2 server endpoints + (``/{server_name}/authorize``, ``/token`` etc.), and ``.well-known`` + discovery routes intentionally fall through — those flows do not need + OAuth2 token passthrough. Clients aggregating multiple servers should + use ``x-mcp-servers``, which takes precedence over path parsing. + """ + segments = [s for s in path.split("/") if s] + if len(segments) >= 2 and segments[0] == "mcp": + return [segments[1]] + if len(segments) >= 2 and segments[1] == "mcp": + return [segments[0]] + return [] + + @staticmethod + def _target_servers_use_oauth2(path: str, mcp_servers: Optional[List[str]]) -> bool: + """ + True only when EVERY MCP server the request targets is configured for + ``auth_type == oauth2``. If any target is non-OAuth2 — or if the target + cannot be resolved at all — return False so the caller fails closed. + + Used to gate the "treat Authorization as opaque OAuth2 token" fallback + in :meth:`process_mcp_request` so a failed LiteLLM-auth cannot be + exchanged for an anonymous session against a non-OAuth2 server. + """ + # Inline imports avoid a circular dependency: mcp_server_manager imports + # from this module. + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + + # Use the x-mcp-servers header verbatim when present (including the + # explicitly-empty list, which means "no targets" → fail closed). + # Only fall back to path parsing when the header was absent entirely. + target_names = ( + mcp_servers + if mcp_servers is not None + else MCPRequestHandler._extract_target_server_names_from_path(path) + ) + if not target_names: + return False + + for name in target_names: + server = global_mcp_server_manager.get_mcp_server_by_name(name) + if server is None or server.auth_type != MCPAuth.oauth2: + return False + return True + @staticmethod def _get_mcp_auth_header_from_headers(headers: Headers) -> Optional[str]: """ diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 21abaa3f981..a6f0d145e9b 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -1,4 +1,5 @@ import base64 +import binascii import json from datetime import datetime, timedelta, timezone from typing import Any, Dict, Iterable, List, Optional, Set, Union, cast @@ -498,6 +499,82 @@ async def rotate_mcp_server_credentials_master_key( ) +def _decode_user_credential(stored: str) -> Optional[str]: + """Read back a value persisted in ``LiteLLM_MCPUserCredentials.credential_b64``. + + Tries nacl decryption first (current write format). Falls back to a + plain ``urlsafe_b64decode`` for rows persisted by older code that wrote + the credential without encryption. Returns ``None`` when neither path + yields a valid string. + """ + decrypted = decrypt_value_helper( + value=stored, + key="mcp_user_credential", + exception_type="debug", + return_original_value=False, + ) + if decrypted is not None: + return decrypted + try: + return base64.urlsafe_b64decode(stored).decode() + except (binascii.Error, UnicodeDecodeError, ValueError, TypeError): + return None + + +def _decode_oauth_payload(stored: str) -> Optional[Dict[str, Any]]: + """Return the OAuth2 payload dict if ``stored`` holds one, else ``None``. + + A row is considered an OAuth2 credential iff its decoded value parses as + a JSON object with ``"type": "oauth2"``. Plain BYOK credentials (which + share the same column) decode to a non-JSON string and return ``None``. + """ + decoded = _decode_user_credential(stored) + if decoded is None: + return None + try: + parsed = json.loads(decoded) + except (ValueError, TypeError): + return None + if isinstance(parsed, dict) and parsed.get("type") == "oauth2": + return parsed + return None + + +async def rotate_mcp_user_credentials_master_key( + prisma_client: PrismaClient, new_master_key: str +): + """Re-encrypt every ``LiteLLM_MCPUserCredentials`` row with ``new_master_key``. + + Reads each ``credential_b64`` with the current salt key (falling back to + legacy plain base64 for unmigrated rows) and writes it back encrypted + under the new master key. Rows that are unreadable under both paths + are logged and skipped so one corrupt row does not abort the rotation. + """ + rows = await prisma_client.db.litellm_mcpusercredentials.find_many() + for row in rows: + plaintext = _decode_user_credential(row.credential_b64) + if plaintext is None: + verbose_proxy_logger.warning( + "rotate_mcp_user_credentials_master_key: could not decode " + "credential for user_id=%s server_id=%s, skipping", + row.user_id, + row.server_id, + ) + continue + re_encrypted = encrypt_value_helper( + plaintext, new_encryption_key=new_master_key + ) + await prisma_client.db.litellm_mcpusercredentials.update( + where={ + "user_id_server_id": { + "user_id": row.user_id, + "server_id": row.server_id, + } + }, + data={"credential_b64": re_encrypted}, + ) + + async def store_user_credential( prisma_client: PrismaClient, user_id: str, @@ -506,7 +583,7 @@ async def store_user_credential( ) -> None: """Store a user credential for a BYOK MCP server.""" - encoded = base64.urlsafe_b64encode(credential.encode()).decode() + encoded = encrypt_value_helper(credential) await prisma_client.db.litellm_mcpusercredentials.upsert( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}, data={ @@ -532,16 +609,7 @@ async def get_user_credential( ) if row is None: return None - try: - return base64.urlsafe_b64decode(row.credential_b64).decode() - except Exception: - # Fall back to nacl decryption for credentials stored by older code - return decrypt_value_helper( - value=row.credential_b64, - key="byok_credential", - exception_type="debug", - return_original_value=False, - ) + return _decode_user_credential(row.credential_b64) async def has_user_credential( @@ -582,7 +650,7 @@ async def store_user_oauth_credential( ) -> None: """Persist an OAuth2 access token for a user+server pair. - The payload is JSON-serialised and stored base64-encoded in the same + The payload is JSON-serialised and stored encrypted in the same ``credential_b64`` column used by BYOK. A ``"type": "oauth2"`` key differentiates it from plain BYOK API keys. """ @@ -606,29 +674,27 @@ async def store_user_oauth_credential( payload["scopes"] = scopes # Guard against silently overwriting a BYOK credential with an OAuth token. - # BYOK credentials lack a "type" field (or use a non-"oauth2" type). # Skip the guard when the caller knows the row is already an OAuth2 credential # (e.g. during token refresh), saving an extra DB round-trip. if not skip_byok_guard: existing = await prisma_client.db.litellm_mcpusercredentials.find_unique( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} ) - if existing is not None: - _byok_error = ValueError( - f"A non-OAuth2 credential already exists for user {user_id} " - f"and server {server_id}. Refusing to overwrite." + if ( + existing is not None + and _decode_oauth_payload(existing.credential_b64) is None + ): + # Existing row is either a BYOK secret or an OAuth2 row that no + # longer decrypts (e.g. after a salt-key rotation). In either + # case, refuse to overwrite — the caller would clobber data + # that may still be recoverable. + raise ValueError( + f"Existing credential for user {user_id} and server " + f"{server_id} could not be verified as an OAuth2 token. " + f"Refusing to overwrite." ) - try: - raw = json.loads( - base64.urlsafe_b64decode(existing.credential_b64).decode() - ) - except Exception: - # Credential is not base64+JSON — it's a plain-text BYOK key. - raise _byok_error - if raw.get("type") != "oauth2": - raise _byok_error - encoded = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode() + encoded = encrypt_value_helper(json.dumps(payload)) await prisma_client.db.litellm_mcpusercredentials.upsert( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}, data={ @@ -672,15 +738,7 @@ async def get_user_oauth_credential( ) if row is None: return None - try: - decoded = base64.urlsafe_b64decode(row.credential_b64).decode() - parsed = json.loads(decoded) - if isinstance(parsed, dict) and parsed.get("type") == "oauth2": - return parsed - # Row exists but is a BYOK (plain string), not an OAuth token - return None - except Exception: - return None + return _decode_oauth_payload(row.credential_b64) async def list_user_oauth_credentials( @@ -694,14 +752,11 @@ async def list_user_oauth_credentials( ) results: List[Dict[str, Any]] = [] for row in rows: - try: - decoded = base64.urlsafe_b64decode(row.credential_b64).decode() - parsed = json.loads(decoded) - if isinstance(parsed, dict) and parsed.get("type") == "oauth2": - parsed["server_id"] = row.server_id - results.append(parsed) - except Exception: - pass # Skip non-OAuth rows (BYOK plain strings) + payload = _decode_oauth_payload(row.credential_b64) + if payload is None: + continue + payload["server_id"] = row.server_id + results.append(payload) return results diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 13b19aa2d65..1794cd14381 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -33,10 +33,12 @@ def get_request_base_url(request: Request) -> str: """ Get the base URL for the request, considering X-Forwarded-* headers. - When behind a proxy (like nginx), the proxy may set: - - X-Forwarded-Proto: The original protocol (http/https) - - X-Forwarded-Host: The original host (may include port) - - X-Forwarded-Port: The original port (if not in Host header) + X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured + when the request comes from a configured trusted proxy + (``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``). + Otherwise the request's literal ``base_url`` is returned, so an + untrusted caller cannot poison OAuth-discovery / redirect_uri values + by injecting headers. Args: request: FastAPI Request object @@ -47,34 +49,28 @@ def get_request_base_url(request: Request) -> str: base_url = str(request.base_url).rstrip("/") parsed = urlparse(base_url) - # Get forwarded headers + if not IPAddressUtils.is_request_from_trusted_proxy(request): + return base_url + x_forwarded_proto = request.headers.get("X-Forwarded-Proto") x_forwarded_host = request.headers.get("X-Forwarded-Host") x_forwarded_port = request.headers.get("X-Forwarded-Port") - # Start with the original scheme scheme = x_forwarded_proto if x_forwarded_proto else parsed.scheme - # Handle host and port if x_forwarded_host: # X-Forwarded-Host may already include port (e.g., "example.com:8080") if ":" in x_forwarded_host and not x_forwarded_host.startswith("["): - # Host includes port netloc = x_forwarded_host elif x_forwarded_port: - # Port is separate netloc = f"{x_forwarded_host}:{x_forwarded_port}" else: - # Just host, no explicit port netloc = x_forwarded_host else: - # No X-Forwarded-Host, use original netloc netloc = parsed.netloc if x_forwarded_port and ":" not in netloc: - # Add forwarded port if not already in netloc netloc = f"{netloc}:{x_forwarded_port}" - # Reconstruct the URL return urlunparse((scheme, netloc, parsed.path, "", "", "")) @@ -131,6 +127,22 @@ def decode_state_hash(encrypted_state: str) -> dict: return state_data +def _get_validated_client_redirect_uri(state_data: Dict[str, Any]) -> str: + """Return a loopback client redirect URI from OAuth state.""" + redirect_uri = state_data.get("client_redirect_uri") or state_data.get("base_url") + if not redirect_uri or not isinstance(redirect_uri, str): + raise HTTPException(status_code=400, detail="Invalid redirect URI") + validate_loopback_redirect_uri(redirect_uri) + return redirect_uri + + +def _append_query_params(url: str, params: Dict[str, str]) -> str: + parsed = urlparse(url) + query_params = parse_qsl(parsed.query, keep_blank_values=True) + query_params.extend(params.items()) + return urlunparse(parsed._replace(query=urlencode(query_params))) + + def _resolve_oauth2_server_for_root_endpoints( client_ip: Optional[str] = None, ) -> Optional[MCPServer]: @@ -568,7 +580,7 @@ async def authorize( else None ) if mcp_server is None and mcp_server_name is None: - mcp_server = _resolve_oauth2_server_for_root_endpoints() + mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) if mcp_server is None: raise HTTPException(status_code=404, detail="MCP server not found") # Use server's stored client_id when caller doesn't supply one. @@ -630,7 +642,7 @@ async def token_endpoint( lookup_name, client_ip=client_ip ) if mcp_server is None and mcp_server_name is None: - mcp_server = _resolve_oauth2_server_for_root_endpoints() + mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) if mcp_server is None: raise HTTPException(status_code=404, detail="MCP server not found") return await exchange_token_with_server( @@ -651,7 +663,6 @@ async def token_endpoint( async def callback(code: str, state: str): try: state_data = decode_state_hash(state) - base_url = state_data["base_url"] original_state = state_data["original_state"] # Re-validate loopback at the sink. /authorize rejects non-loopback @@ -659,10 +670,10 @@ async def callback(code: str, state: str): # minted before that check was added have no expiry and remain # valid indefinitely. Validating here blocks the open-redirect + # code-theft primitive even for pre-fix states. - validate_loopback_redirect_uri(base_url) + redirect_uri = _get_validated_client_redirect_uri(state_data) params = {"code": code, "state": original_state} - complete_returned_url = f"{base_url}?{urlencode(params)}" + complete_returned_url = _append_query_params(redirect_uri, params) return RedirectResponse(url=complete_returned_url, status_code=302) except HTTPException: @@ -719,16 +730,16 @@ def _build_oauth_protected_resource_response( ) request_base_url = get_request_base_url(request) + client_ip = IPAddressUtils.get_mcp_client_ip(request) # When no server name provided, try to resolve the single OAuth2 server if mcp_server_name is None: - resolved = _resolve_oauth2_server_for_root_endpoints() + resolved = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) if resolved: mcp_server_name = resolved.server_name or resolved.name mcp_server: Optional[MCPServer] = None if mcp_server_name: - client_ip = IPAddressUtils.get_mcp_client_ip(request) mcp_server = global_mcp_server_manager.get_mcp_server_by_name( mcp_server_name, client_ip=client_ip ) @@ -835,10 +846,11 @@ def _build_oauth_authorization_server_response( ) request_base_url = get_request_base_url(request) + client_ip = IPAddressUtils.get_mcp_client_ip(request) # When no server name provided, try to resolve the single OAuth2 server if mcp_server_name is None: - resolved = _resolve_oauth2_server_for_root_endpoints() + resolved = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) if resolved: mcp_server_name = resolved.server_name or resolved.name @@ -855,7 +867,6 @@ def _build_oauth_authorization_server_response( mcp_server: Optional[MCPServer] = None if mcp_server_name: - client_ip = IPAddressUtils.get_mcp_client_ip(request) mcp_server = global_mcp_server_manager.get_mcp_server_by_name( mcp_server_name, client_ip=client_ip ) @@ -1007,8 +1018,9 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non "client_secret": "dummy", "redirect_uris": [f"{request_base_url}/callback"], } + client_ip = IPAddressUtils.get_mcp_client_ip(request) if not mcp_server_name: - resolved = _resolve_oauth2_server_for_root_endpoints() + resolved = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip) if resolved: return await register_client_with_server( request=request, @@ -1021,7 +1033,6 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non ) return dummy_return - client_ip = IPAddressUtils.get_mcp_client_ip(request) mcp_server = global_mcp_server_manager.get_mcp_server_by_name( mcp_server_name, client_ip=client_ip ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 25c47aacf0f..9923c3ce4bf 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -41,6 +41,7 @@ from litellm.constants import ( MCP_TOOL_LISTING_TIMEOUT, ) from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException +from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get from litellm.experimental_mcp_client.client import MCPClient, MCPSigV4Auth from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( @@ -168,6 +169,37 @@ def _deserialize_json_dict(data: Any) -> Optional[Dict[str, str]]: class MCPServerManager: _STDIO_ENV_TEMPLATE_PATTERN = re.compile(r"^\$\{(X-[^}]+)\}$") + @staticmethod + def _resolve_oauth2_flow( + *, + auth_type: Optional[MCPAuthType], + oauth2_flow: Optional[str], + token_url: Optional[str], + authorization_url: Optional[str], + client_id: Optional[str], + client_secret: Optional[str], + ) -> Optional[Literal["client_credentials", "authorization_code"]]: + """Infer oauth2_flow for legacy records that omit the field. + + DB rows created before oauth2_flow support may have OAuth2 client + credentials + token_url but a null oauth2_flow. Treat these as M2M, + unless authorization_url is present (interactive OAuth). + """ + if oauth2_flow in ("client_credentials", "authorization_code"): + return cast( + Literal["client_credentials", "authorization_code"], oauth2_flow + ) + if oauth2_flow: + # Ignore unknown/untyped values and continue legacy inference. + return None + if auth_type != MCPAuth.oauth2: + return None + if authorization_url: + return None + if token_url and client_id and client_secret: + return "client_credentials" + return None + def __init__(self): self.registry: Dict[str, MCPServer] = {} self.config_mcp_servers: Dict[str, MCPServer] = {} @@ -341,7 +373,14 @@ class MCPServerManager: # oauth specific fields client_id=server_config.get("client_id", None), client_secret=server_config.get("client_secret", None), - oauth2_flow=server_config.get("oauth2_flow", None), + oauth2_flow=self._resolve_oauth2_flow( + auth_type=auth_type, + oauth2_flow=server_config.get("oauth2_flow", None), + token_url=resolved_token_url, + authorization_url=resolved_authorization_url, + client_id=server_config.get("client_id", None), + client_secret=server_config.get("client_secret", None), + ), scopes=resolved_scopes, authorization_url=resolved_authorization_url, token_url=resolved_token_url, @@ -678,7 +717,17 @@ class MCPServerManager: client_id=client_id_value or getattr(mcp_server, "client_id", None), client_secret=client_secret_value or getattr(mcp_server, "client_secret", None), - oauth2_flow=getattr(mcp_server, "oauth2_flow", None), + oauth2_flow=self._resolve_oauth2_flow( + auth_type=auth_type, + oauth2_flow=getattr(mcp_server, "oauth2_flow", None), + token_url=mcp_server.token_url + or getattr(mcp_oauth_metadata, "token_url", None), + authorization_url=mcp_server.authorization_url + or getattr(mcp_oauth_metadata, "authorization_url", None), + client_id=client_id_value or getattr(mcp_server, "client_id", None), + client_secret=client_secret_value + or getattr(mcp_server, "client_secret", None), + ), scopes=resolved_scopes, authorization_url=mcp_server.authorization_url or getattr(mcp_oauth_metadata, "authorization_url", None), @@ -1499,6 +1548,47 @@ class MCPServerManager: ) return await client.get_prompt(get_prompt_request_params) + @staticmethod + def _is_same_authority_metadata_url(url: str, server_url: str) -> bool: + """ + Whether ``url`` shares scheme, host, and port with ``server_url``. + + Same-authority metadata URLs are produced by our well-known discovery + construction and by resource servers that publish protected-resource + metadata on the resource origin. These must keep working for + administrator-configured internal MCP servers, so they are fetched + directly. Cross-origin URLs are fetched through ``async_safe_get``. + """ + try: + target = urlparse(url) + base = urlparse(server_url) + except Exception: + return False + + if target.scheme not in ("http", "https") or not target.hostname: + return False + + target_port = target.port or (443 if target.scheme == "https" else 80) + base_port = base.port or (443 if base.scheme == "https" else 80) + return ( + base.scheme == target.scheme + and (base.hostname or "").lower() == target.hostname.lower() + and base_port == target_port + ) + + async def _fetch_oauth_discovery_url(self, url: str, server_url: str) -> Any: + client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.MCP, + params={"timeout": MCP_METADATA_TIMEOUT}, + ) + if self._is_same_authority_metadata_url(url, server_url): + # Same-authority URLs may point at administrator-configured + # internal MCP servers. Do not run them through user URL + # validation, but also do not follow redirects because the + # redirect target would not inherit the same-authority guarantee. + return await client.get(url, follow_redirects=False) + return await async_safe_get(client, url) + async def _descovery_metadata( self, server_url: str, @@ -1514,7 +1604,7 @@ class MCPServerManager: resource_scopes, ) = await self._attempt_well_known_discovery(server_url) metadata = await self._fetch_authorization_server_metadata( - authorization_servers + authorization_servers, server_url ) if ( metadata is None @@ -1555,7 +1645,7 @@ class MCPServerManager: authorization_servers, resource_scopes, ) = await self._fetch_oauth_metadata_from_resource( - resource_metadata_url + resource_metadata_url, server_url ) else: ( @@ -1576,7 +1666,7 @@ class MCPServerManager: if authorization_servers: metadata = await self._fetch_authorization_server_metadata( - authorization_servers + authorization_servers, server_url ) preferred_scopes = scopes or resource_scopes @@ -1616,19 +1706,26 @@ class MCPServerManager: return resource_metadata_url, scopes async def _fetch_oauth_metadata_from_resource( - self, resource_metadata_url: str + self, resource_metadata_url: str, server_url: str ) -> Tuple[List[str], Optional[List[str]]]: if not resource_metadata_url: return [], None try: - client = get_async_httpx_client( - llm_provider=httpxSpecialProvider.MCP, - params={"timeout": MCP_METADATA_TIMEOUT}, + response = await self._fetch_oauth_discovery_url( + resource_metadata_url, server_url ) - response = await client.get(resource_metadata_url) response.raise_for_status() data = response.json() + except SSRFError as exc: + verbose_logger.warning( + "MCP OAuth discovery: refusing to fetch resource metadata from %s " + "(rejected by SSRF guard for server %s): %s", + resource_metadata_url, + server_url, + exc, + ) + return [], None except Exception as exc: # pragma: no cover - network issues verbose_logger.debug( "Failed to fetch MCP OAuth metadata from %s: %s", @@ -1677,23 +1774,25 @@ class MCPServerManager: ( authorization_servers, scopes, - ) = await self._fetch_oauth_metadata_from_resource(url) + ) = await self._fetch_oauth_metadata_from_resource(url, server_url) if authorization_servers: return authorization_servers, scopes return [], None async def _fetch_authorization_server_metadata( - self, authorization_servers: List[str] + self, authorization_servers: List[str], server_url: str ) -> Optional[MCPOAuthMetadata]: for issuer in authorization_servers: - metadata = await self._fetch_single_authorization_server_metadata(issuer) + metadata = await self._fetch_single_authorization_server_metadata( + issuer, server_url + ) if metadata is not None: return metadata return None async def _fetch_single_authorization_server_metadata( - self, issuer_url: str + self, issuer_url: str, server_url: str ) -> Optional[MCPOAuthMetadata]: try: parsed = urlparse(issuer_url) @@ -1721,13 +1820,18 @@ class MCPServerManager: for url in candidate_urls: try: - client = get_async_httpx_client( - llm_provider=httpxSpecialProvider.MCP, - params={"timeout": MCP_METADATA_TIMEOUT}, - ) - response = await client.get(url) + response = await self._fetch_oauth_discovery_url(url, server_url) response.raise_for_status() data = response.json() + except SSRFError as exc: + verbose_logger.warning( + "MCP OAuth discovery: refusing to fetch authorization-server " + "metadata from %s (rejected by SSRF guard for server %s): %s", + url, + server_url, + exc, + ) + continue except Exception as exc: # pragma: no cover - network issues verbose_logger.debug( "Failed to fetch authorization metadata from %s: %s", @@ -2370,7 +2474,7 @@ class MCPServerManager: ) ) - async def _call_regular_mcp_tool( + async def _call_regular_mcp_tool( # noqa: PLR0915 self, mcp_server: MCPServer, original_tool_name: str, @@ -2433,7 +2537,11 @@ class MCPServerManager: # oauth2 headers extra_headers: Optional[Dict[str, str]] = None if mcp_server.auth_type == MCPAuth.oauth2: - extra_headers = oauth2_headers + if mcp_server.has_client_credentials: + # For M2M OAuth servers, Authorization must come from token fetch. + extra_headers = None + else: + extra_headers = oauth2_headers if mcp_server.extra_headers and raw_headers: if extra_headers is None: @@ -2445,6 +2553,11 @@ class MCPServerManager: for header in mcp_server.extra_headers: if not isinstance(header, str): continue + if ( + mcp_server.has_client_credentials + and header.lower() == "authorization" + ): + continue header_value = normalized_raw_headers.get(header.lower()) if header_value is None: continue @@ -2480,6 +2593,10 @@ class MCPServerManager: ) extra_headers.update(hook_extra_headers) + # Reset to None if no headers were actually added + if extra_headers is not None and len(extra_headers) == 0: + extra_headers = None + stdio_env = self._build_stdio_env(mcp_server, raw_headers) client = await self._create_mcp_client( diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index ae6055217b8..abb4b5cfa6f 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -153,6 +153,7 @@ if MCP_AVAILABLE: MCPAuthenticatedUser, ) from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, global_mcp_server_manager, ) from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( @@ -900,6 +901,20 @@ if MCP_AVAILABLE: allowed_mcp_server_id ) if mcp_server is not None: + # Apply oauth2_flow resolution for legacy DB rows where it may be NULL + resolved_flow = MCPServerManager._resolve_oauth2_flow( + auth_type=mcp_server.auth_type, + oauth2_flow=mcp_server.oauth2_flow, + token_url=mcp_server.token_url, + authorization_url=mcp_server.authorization_url, + client_id=mcp_server.client_id, + client_secret=mcp_server.client_secret, + ) + if resolved_flow and resolved_flow != mcp_server.oauth2_flow: + # Create a new instance with the resolved flow for this request + mcp_server = mcp_server.model_copy( + update={"oauth2_flow": resolved_flow} + ) allowed_mcp_servers.append(mcp_server) if mcp_servers is not None: @@ -1100,8 +1115,13 @@ if MCP_AVAILABLE: extra_headers: Optional[Dict[str, str]] = None if server.auth_type == MCPAuth.oauth2: - # Copy to avoid mutating the original dict (important for parallel fetching) - extra_headers = oauth2_headers.copy() if oauth2_headers else None + # For OAuth2 M2M servers, upstream Authorization must come from + # client_credentials token fetch, never from caller headers. + if server.has_client_credentials: + extra_headers = None + else: + # Copy to avoid mutating the original dict (important for parallel fetching) + extra_headers = oauth2_headers.copy() if oauth2_headers else None if server.extra_headers and raw_headers: if extra_headers is None: @@ -1114,11 +1134,17 @@ if MCP_AVAILABLE: for header in server.extra_headers: if not isinstance(header, str): continue + if server.has_client_credentials and header.lower() == "authorization": + continue header_value = normalized_raw_headers.get(header.lower()) if header_value is None: continue extra_headers[header] = header_value + # Reset to None if no headers were actually added + if extra_headers is not None and len(extra_headers) == 0: + extra_headers = None + if server_auth_header is None: server_auth_header = mcp_auth_header @@ -1377,11 +1403,19 @@ if MCP_AVAILABLE: spend_meta["per_server_tool_counts"] = per_server_tool_counts end_time = datetime.now() - await litellm_logging_obj.async_success_handler( - result=all_tools, - start_time=list_tools_start_time, - end_time=end_time, - ) + try: + await litellm_logging_obj.async_success_handler( + result=all_tools, + start_time=list_tools_start_time, + end_time=end_time, + ) + except Exception as log_exc: + # list_tools responses must not be dropped due to non-blocking + # observability/serialization failures. + verbose_logger.warning( + "MCP list_tools success logging failed (continuing): %s", + log_exc, + ) verbose_logger.info( f"Successfully fetched {len(all_tools)} tools total from all MCP servers" diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404/index.html similarity index 100% rename from litellm/proxy/_experimental/out/404.html rename to litellm/proxy/_experimental/out/404/index.html diff --git a/litellm/proxy/_experimental/out/_not-found.html b/litellm/proxy/_experimental/out/_not-found/index.html similarity index 100% rename from litellm/proxy/_experimental/out/_not-found.html rename to litellm/proxy/_experimental/out/_not-found/index.html diff --git a/litellm/proxy/_experimental/out/api-reference.html b/litellm/proxy/_experimental/out/api-reference/index.html similarity index 100% rename from litellm/proxy/_experimental/out/api-reference.html rename to litellm/proxy/_experimental/out/api-reference/index.html diff --git a/litellm/proxy/_experimental/out/experimental/api-playground.html b/litellm/proxy/_experimental/out/experimental/api-playground/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/api-playground.html rename to litellm/proxy/_experimental/out/experimental/api-playground/index.html diff --git a/litellm/proxy/_experimental/out/experimental/budgets.html b/litellm/proxy/_experimental/out/experimental/budgets/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/budgets.html rename to litellm/proxy/_experimental/out/experimental/budgets/index.html diff --git a/litellm/proxy/_experimental/out/experimental/caching.html b/litellm/proxy/_experimental/out/experimental/caching/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/caching.html rename to litellm/proxy/_experimental/out/experimental/caching/index.html diff --git a/litellm/proxy/_experimental/out/experimental/claude-code-plugins.html b/litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/claude-code-plugins.html rename to litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html diff --git a/litellm/proxy/_experimental/out/experimental/old-usage.html b/litellm/proxy/_experimental/out/experimental/old-usage/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/old-usage.html rename to litellm/proxy/_experimental/out/experimental/old-usage/index.html diff --git a/litellm/proxy/_experimental/out/experimental/prompts.html b/litellm/proxy/_experimental/out/experimental/prompts/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/prompts.html rename to litellm/proxy/_experimental/out/experimental/prompts/index.html diff --git a/litellm/proxy/_experimental/out/experimental/tag-management.html b/litellm/proxy/_experimental/out/experimental/tag-management/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/tag-management.html rename to litellm/proxy/_experimental/out/experimental/tag-management/index.html diff --git a/litellm/proxy/_experimental/out/guardrails.html b/litellm/proxy/_experimental/out/guardrails/index.html similarity index 100% rename from litellm/proxy/_experimental/out/guardrails.html rename to litellm/proxy/_experimental/out/guardrails/index.html diff --git a/litellm/proxy/_experimental/out/login.html b/litellm/proxy/_experimental/out/login/index.html similarity index 100% rename from litellm/proxy/_experimental/out/login.html rename to litellm/proxy/_experimental/out/login/index.html diff --git a/litellm/proxy/_experimental/out/logs.html b/litellm/proxy/_experimental/out/logs/index.html similarity index 100% rename from litellm/proxy/_experimental/out/logs.html rename to litellm/proxy/_experimental/out/logs/index.html diff --git a/litellm/proxy/_experimental/out/mcp/oauth/callback.html b/litellm/proxy/_experimental/out/mcp/oauth/callback/index.html similarity index 100% rename from litellm/proxy/_experimental/out/mcp/oauth/callback.html rename to litellm/proxy/_experimental/out/mcp/oauth/callback/index.html diff --git a/litellm/proxy/_experimental/out/model-hub.html b/litellm/proxy/_experimental/out/model-hub/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model-hub.html rename to litellm/proxy/_experimental/out/model-hub/index.html diff --git a/litellm/proxy/_experimental/out/model_hub.html b/litellm/proxy/_experimental/out/model_hub/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub.html rename to litellm/proxy/_experimental/out/model_hub/index.html diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/models-and-endpoints.html b/litellm/proxy/_experimental/out/models-and-endpoints/index.html similarity index 100% rename from litellm/proxy/_experimental/out/models-and-endpoints.html rename to litellm/proxy/_experimental/out/models-and-endpoints/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding/index.html similarity index 100% rename from litellm/proxy/_experimental/out/onboarding.html rename to litellm/proxy/_experimental/out/onboarding/index.html diff --git a/litellm/proxy/_experimental/out/organizations.html b/litellm/proxy/_experimental/out/organizations/index.html similarity index 100% rename from litellm/proxy/_experimental/out/organizations.html rename to litellm/proxy/_experimental/out/organizations/index.html diff --git a/litellm/proxy/_experimental/out/playground.html b/litellm/proxy/_experimental/out/playground/index.html similarity index 100% rename from litellm/proxy/_experimental/out/playground.html rename to litellm/proxy/_experimental/out/playground/index.html diff --git a/litellm/proxy/_experimental/out/policies.html b/litellm/proxy/_experimental/out/policies/index.html similarity index 100% rename from litellm/proxy/_experimental/out/policies.html rename to litellm/proxy/_experimental/out/policies/index.html diff --git a/litellm/proxy/_experimental/out/settings/admin-settings.html b/litellm/proxy/_experimental/out/settings/admin-settings/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/admin-settings.html rename to litellm/proxy/_experimental/out/settings/admin-settings/index.html diff --git a/litellm/proxy/_experimental/out/settings/logging-and-alerts.html b/litellm/proxy/_experimental/out/settings/logging-and-alerts/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/logging-and-alerts.html rename to litellm/proxy/_experimental/out/settings/logging-and-alerts/index.html diff --git a/litellm/proxy/_experimental/out/settings/router-settings.html b/litellm/proxy/_experimental/out/settings/router-settings/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/router-settings.html rename to litellm/proxy/_experimental/out/settings/router-settings/index.html diff --git a/litellm/proxy/_experimental/out/settings/ui-theme.html b/litellm/proxy/_experimental/out/settings/ui-theme/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/ui-theme.html rename to litellm/proxy/_experimental/out/settings/ui-theme/index.html diff --git a/litellm/proxy/_experimental/out/skills.html b/litellm/proxy/_experimental/out/skills/index.html similarity index 100% rename from litellm/proxy/_experimental/out/skills.html rename to litellm/proxy/_experimental/out/skills/index.html diff --git a/litellm/proxy/_experimental/out/teams.html b/litellm/proxy/_experimental/out/teams/index.html similarity index 100% rename from litellm/proxy/_experimental/out/teams.html rename to litellm/proxy/_experimental/out/teams/index.html diff --git a/litellm/proxy/_experimental/out/test-key.html b/litellm/proxy/_experimental/out/test-key/index.html similarity index 100% rename from litellm/proxy/_experimental/out/test-key.html rename to litellm/proxy/_experimental/out/test-key/index.html diff --git a/litellm/proxy/_experimental/out/tools/mcp-servers.html b/litellm/proxy/_experimental/out/tools/mcp-servers/index.html similarity index 100% rename from litellm/proxy/_experimental/out/tools/mcp-servers.html rename to litellm/proxy/_experimental/out/tools/mcp-servers/index.html diff --git a/litellm/proxy/_experimental/out/tools/vector-stores.html b/litellm/proxy/_experimental/out/tools/vector-stores/index.html similarity index 100% rename from litellm/proxy/_experimental/out/tools/vector-stores.html rename to litellm/proxy/_experimental/out/tools/vector-stores/index.html diff --git a/litellm/proxy/_experimental/out/usage.html b/litellm/proxy/_experimental/out/usage/index.html similarity index 100% rename from litellm/proxy/_experimental/out/usage.html rename to litellm/proxy/_experimental/out/usage/index.html diff --git a/litellm/proxy/_experimental/out/users.html b/litellm/proxy/_experimental/out/users/index.html similarity index 100% rename from litellm/proxy/_experimental/out/users.html rename to litellm/proxy/_experimental/out/users/index.html diff --git a/litellm/proxy/_experimental/out/virtual-keys.html b/litellm/proxy/_experimental/out/virtual-keys/index.html similarity index 100% rename from litellm/proxy/_experimental/out/virtual-keys.html rename to litellm/proxy/_experimental/out/virtual-keys/index.html diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py new file mode 100644 index 00000000000..d58bfbbdb52 --- /dev/null +++ b/litellm/proxy/_lazy_features.py @@ -0,0 +1,432 @@ +""" +Lazy registration for optional feature routers. Each LAZY_FEATURES entry +imports its module only on the first request matching its path prefix, +saving ~700 MB at idle for deployments that don't use these features. +First hit pays the import cost (1-3 s for heavy modules); /openapi.json +omits each feature's routes until the feature is warmed. +""" + +import asyncio +import importlib +import sys +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Callable, Dict, Tuple + +from starlette.types import Receive, Scope, Send + +from litellm._logging import verbose_proxy_logger + +if TYPE_CHECKING: + from fastapi import APIRouter, FastAPI + + +def _include_router(attr_name: str = "router") -> Callable[["FastAPI", object], None]: + def _register(app: "FastAPI", module: object) -> None: + app.include_router(getattr(module, attr_name)) + + return _register + + +def _mount_app( + prefix: str, attr_name: str = "app" +) -> Callable[["FastAPI", object], None]: + def _register(app: "FastAPI", module: object) -> None: + app.mount(path=prefix, app=getattr(module, attr_name)) + + return _register + + +@dataclass(frozen=True) +class LazyFeature: + name: str + module_path: str + path_prefixes: Tuple[str, ...] + register_fn: Callable[["FastAPI", object], None] = field( + default_factory=lambda: _include_router("router") + ) + # For routes whose path has a leading parameter (e.g. /{server}/authorize) + # — startswith can't match those, so the matcher also checks endswith. + path_suffixes: Tuple[str, ...] = () + # Keep the stub injected even after load — for mounted ASGI sub-apps + # whose routes don't appear in the parent app's openapi spec. + persistent_swagger_stub: bool = False + + +LAZY_FEATURES: Tuple[LazyFeature, ...] = ( + LazyFeature( + name="guardrails", + module_path="litellm.proxy.guardrails.guardrail_endpoints", + path_prefixes=( + "/guardrails", + "/v2/guardrails", + "/apply_guardrail", + "/policies/usage", + ), + ), + LazyFeature( + name="policies", + module_path="litellm.proxy.management_endpoints.policy_endpoints", + # Trailing slash to avoid matching /policies/... (policy_engine). + path_prefixes=("/policy/", "/utils/test_policies_and_guardrails"), + ), + LazyFeature( + name="policy_engine", + module_path="litellm.proxy.policy_engine.policy_endpoints", + path_prefixes=("/policies",), + ), + LazyFeature( + name="policy_resolve", + module_path="litellm.proxy.policy_engine.policy_resolve_endpoints", + path_prefixes=("/policies/resolve", "/policies/attachments/estimate-impact"), + ), + LazyFeature( + name="agents", + module_path="litellm.proxy.agent_endpoints.endpoints", + path_prefixes=("/v1/agents", "/agents", "/agent/"), + ), + LazyFeature( + name="a2a", + module_path="litellm.proxy.agent_endpoints.a2a_endpoints", + path_prefixes=("/a2a", "/v1/a2a"), + ), + LazyFeature( + name="vector_stores", + module_path="litellm.proxy.vector_store_endpoints.endpoints", + path_prefixes=("/v1/vector_stores", "/vector_stores", "/v1/indexes"), + ), + LazyFeature( + name="vector_store_management", + module_path="litellm.proxy.vector_store_endpoints.management_endpoints", + # Trailing slash to avoid matching /vector_stores/... (vector_stores). + path_prefixes=("/vector_store/", "/v1/vector_store/"), + ), + LazyFeature( + name="vector_store_files", + # Routes appear under both /v1/vector_stores/{id}/files and the + # un-versioned form, so both prefixes must trigger the load. + module_path="litellm.proxy.vector_store_files_endpoints.endpoints", + path_prefixes=("/v1/vector_stores", "/vector_stores"), + ), + LazyFeature( + name="tools", + module_path="litellm.proxy.management_endpoints.tool_management_endpoints", + path_prefixes=("/v1/tool", "/tool"), + ), + LazyFeature( + name="search_tools", + module_path="litellm.proxy.search_endpoints.search_tool_management", + path_prefixes=("/search_tools",), + ), + # mcp_management owns most /v1/mcp/* admin routes; mcp_app is the mounted + # streaming sub-app at /mcp. + LazyFeature( + name="mcp_management", + module_path="litellm.proxy.management_endpoints.mcp_management_endpoints", + path_prefixes=("/v1/mcp/",), + ), + LazyFeature( + # Also serves /.well-known/oauth-* (OAuth metadata discovery). + # No /mcp/oauth prefix here: the mounted /mcp sub-app would + # shadow it, and there are no actual routes there anyway. + name="mcp_byok_oauth", + module_path="litellm.proxy._experimental.mcp_server.byok_oauth_endpoints", + path_prefixes=("/v1/mcp/oauth", "/.well-known/oauth-"), + ), + LazyFeature( + # Serves OAuth dance endpoints (/authorize, /token, /callback, + # /register) plus several /.well-known/ discovery URLs at the proxy + # root — needed for MCP-over-OAuth flows even before /mcp is hit. + name="mcp_discoverable", + module_path="litellm.proxy._experimental.mcp_server.discoverable_endpoints", + path_prefixes=( + "/.well-known/oauth-", + "/.well-known/openid-configuration", + "/.well-known/jwks.json", + "/authorize", + "/token", + "/callback", + "/register", + ), + # Catches the /{mcp_server_name}/authorize|token|register variants. + path_suffixes=("/authorize", "/token", "/register"), + ), + LazyFeature( + name="mcp_rest", + module_path="litellm.proxy._experimental.mcp_server.rest_endpoints", + path_prefixes=("/mcp-rest",), + ), + LazyFeature( + # Hardcoded /mcp matches BASE_MCP_ROUTE; importing the constant + # here would defeat lazy loading. + name="mcp_app", + module_path="litellm.proxy._experimental.mcp_server.server", + path_prefixes=("/mcp",), + register_fn=_mount_app("/mcp", attr_name="app"), + persistent_swagger_stub=True, + ), + LazyFeature( + name="config_overrides", + module_path="litellm.proxy.management_endpoints.config_override_endpoints", + path_prefixes=("/config_overrides",), + ), + LazyFeature( + name="realtime", + module_path="litellm.proxy.realtime_endpoints.endpoints", + path_prefixes=("/openai/v1/realtime", "/v1/realtime", "/realtime"), + ), + LazyFeature( + name="anthropic_passthrough", + module_path="litellm.proxy.anthropic_endpoints.endpoints", + path_prefixes=("/v1/messages", "/anthropic", "/api/event_logging"), + ), + LazyFeature( + name="anthropic_skills", + module_path="litellm.proxy.anthropic_endpoints.skills_endpoints", + path_prefixes=("/v1/skills", "/skills"), + ), + LazyFeature( + name="langfuse_passthrough", + module_path="litellm.proxy.vertex_ai_endpoints.langfuse_endpoints", + path_prefixes=("/langfuse",), + ), + LazyFeature( + name="evals", + module_path="litellm.proxy.openai_evals_endpoints.endpoints", + path_prefixes=("/v1/evals", "/evals"), + ), + LazyFeature( + name="claude_code_marketplace", + module_path="litellm.proxy.anthropic_endpoints.claude_code_endpoints", + path_prefixes=("/claude-code",), + register_fn=_include_router("claude_code_marketplace_router"), + ), + LazyFeature( + name="scim", + module_path="litellm.proxy.management_endpoints.scim.scim_v2", + path_prefixes=("/scim",), + register_fn=_include_router("scim_router"), + ), + LazyFeature( + name="cloudzero", + module_path="litellm.proxy.spend_tracking.cloudzero_endpoints", + path_prefixes=("/cloudzero",), + ), + LazyFeature( + name="vantage", + module_path="litellm.proxy.spend_tracking.vantage_endpoints", + path_prefixes=("/vantage",), + ), + LazyFeature( + name="usage_ai", + module_path="litellm.proxy.management_endpoints.usage_endpoints", + path_prefixes=("/usage/ai",), + ), + LazyFeature( + name="prompts", + module_path="litellm.proxy.prompts.prompt_endpoints", + path_prefixes=("/prompts", "/utils/dotprompt_json_converter"), + ), + LazyFeature( + name="jwt_mappings", + module_path="litellm.proxy.management_endpoints.jwt_key_mapping_endpoints", + path_prefixes=("/jwt/key/mapping",), + ), + LazyFeature( + name="compliance", + module_path="litellm.proxy.management_endpoints.compliance_endpoints", + path_prefixes=("/compliance",), + ), + LazyFeature( + name="access_groups", + module_path="litellm.proxy.management_endpoints.access_group_endpoints", + path_prefixes=("/access_group", "/v1/access_group", "/v1/unified_access_group"), + ), +) + + +class LazyFeatureMiddleware: + """ASGI middleware that imports + registers a feature router on first + matching request. Idempotent; once loaded, subsequent requests skip.""" + + def __init__( + self, + app, + fastapi_app: "FastAPI", + features: Tuple[LazyFeature, ...] = LAZY_FEATURES, + ): + self.app = app + self._fastapi_app = fastapi_app + self._features = features + # Loaded set / per-feature locks live on app.state so the warm endpoint + # and the middleware share them — preventing duplicate registrations + # when both paths fire for the same feature. + if not hasattr(fastapi_app.state, "lazy_loaded"): + fastapi_app.state.lazy_loaded = set() + fastapi_app.state.lazy_locks = {} + + @property + def _loaded(self) -> set: + return self._fastapi_app.state.lazy_loaded + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + # Short-circuit once every feature has loaded. + if scope["type"] in ("http", "websocket") and len(self._loaded) < len( + self._features + ): + path = scope.get("path", "") + for feat in self._features: + if feat.module_path in self._loaded: + continue + if any(path.startswith(p) for p in feat.path_prefixes) or any( + path.endswith(s) for s in feat.path_suffixes + ): + await _force_load(self._fastapi_app, feat) + await self.app(scope, receive, send) + + +async def _force_load(app: "FastAPI", feat: LazyFeature) -> bool: + """Import + register a lazy feature exactly once per (app, module). + Shared by the middleware and the /lazy/warm endpoint.""" + if not hasattr(app.state, "lazy_loaded"): + app.state.lazy_loaded = set() + app.state.lazy_locks = {} + lock = app.state.lazy_locks.setdefault(feat.module_path, asyncio.Lock()) + async with lock: + if feat.module_path in app.state.lazy_loaded: + return False + try: + # Import on a thread (heavy modules take 1-3 s). register_fn + # mutates app.router.routes, so it stays on the loop thread. + loop = asyncio.get_running_loop() + module = await loop.run_in_executor( + None, importlib.import_module, feat.module_path + ) + feat.register_fn(app, module) + app.state.lazy_loaded.add(feat.module_path) + app.openapi_schema = None + verbose_proxy_logger.info( + "Lazy-loaded optional feature %r (module: %s)", + feat.name, + feat.module_path, + ) + return True + except Exception as exc: + # Mark loaded anyway so we don't retry on every request. + app.state.lazy_loaded.add(feat.module_path) + verbose_proxy_logger.warning( + "Failed to lazy-load optional feature %r (module: %s): %s. " + "This feature's endpoints will return 404 until restart.", + feat.name, + feat.module_path, + exc, + ) + return False + + +def attach_lazy_features(app: "FastAPI") -> None: + app.include_router(_make_warmup_router(app)) + app.add_middleware(LazyFeatureMiddleware, fastapi_app=app) + + +def _make_warmup_router(app: "FastAPI") -> "APIRouter": + """POST /lazy/warm/{name}: load a feature and return its partial openapi + so the Swagger plugin can merge in-place without a full /openapi.json refetch. + Requires auth — anyone who can hit the proxy can already trigger the same + imports by sending a real request to a feature's prefix, but gating this + debug endpoint avoids unauthenticated callers forcing the import chain.""" + from fastapi import APIRouter, Depends, HTTPException + from fastapi.openapi.utils import get_openapi + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + router = APIRouter() + + @router.post( + "/lazy/warm/{name}", + include_in_schema=False, + dependencies=[Depends(user_api_key_auth)], + ) + async def warm(name: str): + feat = next((f for f in LAZY_FEATURES if f.name == name), None) + if feat is None: + raise HTTPException(404, f"unknown lazy feature: {name}") + if feat.persistent_swagger_stub: + return {"stub_path": None, "paths": {}, "components": {"schemas": {}}} + + await _force_load(app, feat) + + feat_routes = [ + r + for r in app.routes + if any(getattr(r, "path", "").startswith(p) for p in feat.path_prefixes) + ] + full = get_openapi(title=app.title, version=app.version, routes=feat_routes) + # Force all operations under one tag so they group under a single Swagger + # section — many lazy modules tag routes inconsistently. + for path_ops in full.get("paths", {}).values(): + for op in path_ops.values(): + if isinstance(op, dict): + op["tags"] = [feat.name] + return { + "stub_path": feat.path_prefixes[0], + "paths": full.get("paths", {}), + "components": {"schemas": full.get("components", {}).get("schemas", {})}, + } + + return router + + +def inject_lazy_stubs(schema: Dict) -> Dict: + """Inject openapi entries for unloaded features. Uses the snapshot file + when available (full route info), otherwise falls back to a single + placeholder per feature. Any failure logs and returns the schema unchanged + so /openapi.json never 500s on a cosmetic injection bug.""" + try: + from litellm.proxy._lazy_openapi_snapshot import load_snapshot + + snapshot = load_snapshot() + paths = schema.setdefault("paths", {}) + schemas = schema.setdefault("components", {}).setdefault("schemas", {}) + + for feat in LAZY_FEATURES: + if feat.module_path in sys.modules and not feat.persistent_swagger_stub: + continue + + fragment = (snapshot or {}).get(feat.name) + if fragment: + for p, ops in fragment.get("paths", {}).items(): + paths.setdefault(p, ops) + for name, sch in ( + fragment.get("components", {}).get("schemas", {}).items() + ): + schemas.setdefault(name, sch) + continue + + prefix = feat.path_prefixes[0] + if prefix in paths: + continue + paths[prefix] = { + "get": { + "tags": [feat.name], + "summary": feat.name, + "responses": {"200": {"description": "OK"}}, + } + } + except Exception as exc: + verbose_proxy_logger.warning("inject_lazy_stubs failed: %s", exc) + return schema + + +def lazy_tag_to_prefix() -> Dict[str, str]: + """feature.name -> first prefix, used by the Swagger warmup JS plugin. + Returns empty when the snapshot is loaded — the plugin is unnecessary + because /openapi.json already has full route info.""" + from litellm.proxy._lazy_openapi_snapshot import load_snapshot + + if load_snapshot(): + return {} + return { + feat.name: feat.path_prefixes[0] + for feat in LAZY_FEATURES + if not feat.persistent_swagger_stub + } diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json new file mode 100644 index 00000000000..46a514c0870 --- /dev/null +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -0,0 +1,31651 @@ +{ + "a2a": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/a2a/{agent_id}": { + "post": { + "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response", + "operationId": "invoke_agent_a2a_a2a__agent_id__post", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Invoke Agent A2A", + "tags": [ + "a2a" + ] + } + }, + "/a2a/{agent_id}/.well-known/agent-card.json": { + "get": { + "description": "Get the agent card for an agent (A2A discovery endpoint).\n\nSupports both standard paths:\n- /.well-known/agent-card.json\n- /.well-known/agent.json\n\nThe URL in the agent card is rewritten to point to the LiteLLM proxy,\nso all subsequent A2A calls go through LiteLLM for logging and cost tracking.", + "operationId": "get_agent_card_a2a__agent_id___well_known_agent_card_json_get", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Agent Card", + "tags": [ + "a2a" + ] + } + }, + "/a2a/{agent_id}/.well-known/agent.json": { + "get": { + "description": "Get the agent card for an agent (A2A discovery endpoint).\n\nSupports both standard paths:\n- /.well-known/agent-card.json\n- /.well-known/agent.json\n\nThe URL in the agent card is rewritten to point to the LiteLLM proxy,\nso all subsequent A2A calls go through LiteLLM for logging and cost tracking.", + "operationId": "get_agent_card_a2a__agent_id___well_known_agent_json_get", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Agent Card", + "tags": [ + "a2a" + ] + } + }, + "/a2a/{agent_id}/message/send": { + "post": { + "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response", + "operationId": "invoke_agent_a2a_a2a__agent_id__message_send_post", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Invoke Agent A2A", + "tags": [ + "a2a" + ] + } + }, + "/v1/a2a/{agent_id}/message/send": { + "post": { + "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response", + "operationId": "invoke_agent_a2a_v1_a2a__agent_id__message_send_post", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Invoke Agent A2A", + "tags": [ + "a2a" + ] + } + } + } + }, + "access_groups": { + "components": { + "schemas": { + "AccessGroupCreateRequest": { + "properties": { + "access_agent_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Agent Ids" + }, + "access_group_name": { + "title": "Access Group Name", + "type": "string" + }, + "access_mcp_server_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Mcp Server Ids" + }, + "access_model_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Model Names" + }, + "assigned_key_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Assigned Key Ids" + }, + "assigned_team_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Assigned Team Ids" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + } + }, + "required": [ + "access_group_name" + ], + "title": "AccessGroupCreateRequest", + "type": "object" + }, + "AccessGroupInfo": { + "properties": { + "access_group": { + "title": "Access Group", + "type": "string" + }, + "deployment_count": { + "title": "Deployment Count", + "type": "integer" + }, + "model_names": { + "items": { + "type": "string" + }, + "title": "Model Names", + "type": "array" + } + }, + "required": [ + "access_group", + "model_names", + "deployment_count" + ], + "title": "AccessGroupInfo", + "type": "object" + }, + "AccessGroupResponse": { + "properties": { + "access_agent_ids": { + "items": { + "type": "string" + }, + "title": "Access Agent Ids", + "type": "array" + }, + "access_group_id": { + "title": "Access Group Id", + "type": "string" + }, + "access_group_name": { + "title": "Access Group Name", + "type": "string" + }, + "access_mcp_server_ids": { + "items": { + "type": "string" + }, + "title": "Access Mcp Server Ids", + "type": "array" + }, + "access_model_names": { + "items": { + "type": "string" + }, + "title": "Access Model Names", + "type": "array" + }, + "assigned_key_ids": { + "items": { + "type": "string" + }, + "title": "Assigned Key Ids", + "type": "array" + }, + "assigned_team_ids": { + "items": { + "type": "string" + }, + "title": "Assigned Team Ids", + "type": "array" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated By" + } + }, + "required": [ + "access_group_id", + "access_group_name", + "access_model_names", + "access_mcp_server_ids", + "access_agent_ids", + "assigned_team_ids", + "assigned_key_ids", + "created_at", + "updated_at" + ], + "title": "AccessGroupResponse", + "type": "object" + }, + "AccessGroupUpdateRequest": { + "properties": { + "access_agent_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Agent Ids" + }, + "access_group_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Access Group Name" + }, + "access_mcp_server_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Mcp Server Ids" + }, + "access_model_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Model Names" + }, + "assigned_key_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Assigned Key Ids" + }, + "assigned_team_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Assigned Team Ids" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + } + }, + "title": "AccessGroupUpdateRequest", + "type": "object" + }, + "DeleteModelGroupResponse": { + "properties": { + "access_group": { + "title": "Access Group", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "models_updated": { + "title": "Models Updated", + "type": "integer" + } + }, + "required": [ + "access_group", + "models_updated", + "message" + ], + "title": "DeleteModelGroupResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListAccessGroupsResponse": { + "properties": { + "access_groups": { + "items": { + "$ref": "#/components/schemas/AccessGroupInfo" + }, + "title": "Access Groups", + "type": "array" + } + }, + "required": [ + "access_groups" + ], + "title": "ListAccessGroupsResponse", + "type": "object" + }, + "NewModelGroupRequest": { + "properties": { + "access_group": { + "title": "Access Group", + "type": "string" + }, + "model_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Ids" + }, + "model_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Names" + } + }, + "required": [ + "access_group" + ], + "title": "NewModelGroupRequest", + "type": "object" + }, + "NewModelGroupResponse": { + "properties": { + "access_group": { + "title": "Access Group", + "type": "string" + }, + "model_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Ids" + }, + "model_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Names" + }, + "models_updated": { + "title": "Models Updated", + "type": "integer" + } + }, + "required": [ + "access_group", + "models_updated" + ], + "title": "NewModelGroupResponse", + "type": "object" + }, + "UpdateModelGroupRequest": { + "properties": { + "model_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Ids" + }, + "model_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Model Names" + } + }, + "title": "UpdateModelGroupRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/access_group/list": { + "get": { + "description": "List all access groups.\n\nReturns a list of all access groups with their model names and deployment counts.\n\nExample:\n```bash\ncurl -X GET 'http://localhost:4000/access_group/list' \\\n -H 'Authorization: Bearer sk-1234'\n```\n\nReturns:\n- ListAccessGroupsResponse with all access groups", + "operationId": "list_access_groups_access_group_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListAccessGroupsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Access Groups", + "tags": [ + "access_groups" + ] + } + }, + "/access_group/new": { + "post": { + "description": "Create a new access group containing multiple model names.\n\nAn access group is a named collection of model groups that can be referenced\nby teams/keys for simplified access control.\n\nExample:\n```bash\ncurl -X POST 'http://localhost:4000/access_group/new' \\\n -H 'Authorization: Bearer sk-1234' \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"access_group\": \"production-models\",\n \"model_names\": [\"gpt-4\", \"claude-3-opus\", \"gemini-pro\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (e.g., \"production-models\")\n- model_names: List[str] - List of existing model groups to include\n\nReturns:\n- NewModelGroupResponse with the created access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 500: If database operations fail", + "operationId": "create_model_group_access_group_new_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewModelGroupRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewModelGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Model Group", + "tags": [ + "access_groups" + ] + } + }, + "/access_group/{access_group}/delete": { + "delete": { + "description": "Delete an access group.\n\nRemoves the access group from all deployments that have it.\n\nExample:\n```bash\ncurl -X DELETE 'http://localhost:4000/access_group/production-models/delete' \\\n -H 'Authorization: Bearer sk-1234'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n\nReturns:\n- DeleteModelGroupResponse with deletion details\n\nRaises:\n- HTTPException 404: If access group not found", + "operationId": "delete_access_group_access_group__access_group__delete_delete", + "parameters": [ + { + "in": "path", + "name": "access_group", + "required": true, + "schema": { + "title": "Access Group", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteModelGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Access Group", + "tags": [ + "access_groups" + ] + } + }, + "/access_group/{access_group}/info": { + "get": { + "description": "Get information about a specific access group.\n\nExample:\n```bash\ncurl -X GET 'http://localhost:4000/access_group/production-models/info' \\\n -H 'Authorization: Bearer sk-1234'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n\nReturns:\n- AccessGroupInfo with the access group details\n\nRaises:\n- HTTPException 404: If access group not found", + "operationId": "get_access_group_info_access_group__access_group__info_get", + "parameters": [ + { + "in": "path", + "name": "access_group", + "required": true, + "schema": { + "title": "Access Group", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupInfo" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Access Group Info", + "tags": [ + "access_groups" + ] + } + }, + "/access_group/{access_group}/update": { + "put": { + "description": "Update an access group's model names.\n\nThis will:\n1. Remove the access group from all current deployments\n2. Add the access group to all deployments for the new model_names list\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H 'Authorization: Bearer sk-1234' \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found", + "operationId": "update_access_group_access_group__access_group__update_put", + "parameters": [ + { + "in": "path", + "name": "access_group", + "required": true, + "schema": { + "title": "Access Group", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateModelGroupRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewModelGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Access Group", + "tags": [ + "access_groups" + ] + } + }, + "/v1/access_group": { + "get": { + "operationId": "list_access_groups_v1_access_group_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/AccessGroupResponse" + }, + "title": "Response List Access Groups V1 Access Group Get", + "type": "array" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Access Groups", + "tags": [ + "access_groups" + ] + }, + "post": { + "operationId": "create_access_group_v1_access_group_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Access Group", + "tags": [ + "access_groups" + ] + } + }, + "/v1/access_group/{access_group_id}": { + "delete": { + "operationId": "delete_access_group_v1_access_group__access_group_id__delete", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Access Group", + "tags": [ + "access_groups" + ] + }, + "get": { + "operationId": "get_access_group_v1_access_group__access_group_id__get", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Access Group", + "tags": [ + "access_groups" + ] + }, + "put": { + "operationId": "update_access_group_v1_access_group__access_group_id__put", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Access Group", + "tags": [ + "access_groups" + ] + } + }, + "/v1/unified_access_group": { + "get": { + "operationId": "list_access_groups_v1_unified_access_group_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/AccessGroupResponse" + }, + "title": "Response List Access Groups V1 Unified Access Group Get", + "type": "array" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Access Groups", + "tags": [ + "access_groups" + ] + }, + "post": { + "operationId": "create_access_group_v1_unified_access_group_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Access Group", + "tags": [ + "access_groups" + ] + } + }, + "/v1/unified_access_group/{access_group_id}": { + "delete": { + "operationId": "delete_access_group_v1_unified_access_group__access_group_id__delete", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Access Group", + "tags": [ + "access_groups" + ] + }, + "get": { + "operationId": "get_access_group_v1_unified_access_group__access_group_id__get", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Access Group", + "tags": [ + "access_groups" + ] + }, + "put": { + "operationId": "update_access_group_v1_unified_access_group__access_group_id__put", + "parameters": [ + { + "in": "path", + "name": "access_group_id", + "required": true, + "schema": { + "title": "Access Group Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AccessGroupResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Access Group", + "tags": [ + "access_groups" + ] + } + } + } + }, + "agents": { + "components": { + "schemas": { + "APIKeySecurityScheme": { + "description": "Defines a security scheme using an API key.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "in_": { + "enum": [ + "query", + "header", + "cookie" + ], + "title": "In", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "type": { + "const": "apiKey", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "in_", + "name" + ], + "title": "APIKeySecurityScheme", + "type": "object" + }, + "AgentCapabilities": { + "description": "Defines optional capabilities supported by an agent.", + "properties": { + "extensions": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/AgentExtension" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extensions" + }, + "pushNotifications": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Pushnotifications" + }, + "stateTransitionHistory": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Statetransitionhistory" + }, + "streaming": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Streaming" + } + }, + "title": "AgentCapabilities", + "type": "object" + }, + "AgentCard": { + "description": "The AgentCard is a self-describing manifest for an agent.\nIt provides essential metadata including the agent's identity, capabilities,\nskills, supported communication methods, and security requirements.", + "properties": { + "additionalInterfaces": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/AgentInterface" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Additionalinterfaces" + }, + "capabilities": { + "$ref": "#/components/schemas/AgentCapabilities" + }, + "defaultInputModes": { + "items": { + "type": "string" + }, + "title": "Defaultinputmodes", + "type": "array" + }, + "defaultOutputModes": { + "items": { + "type": "string" + }, + "title": "Defaultoutputmodes", + "type": "array" + }, + "description": { + "title": "Description", + "type": "string" + }, + "documentationUrl": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Documentationurl" + }, + "iconUrl": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Iconurl" + }, + "name": { + "title": "Name", + "type": "string" + }, + "preferredTransport": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Preferredtransport" + }, + "protocolVersion": { + "title": "Protocolversion", + "type": "string" + }, + "provider": { + "anyOf": [ + { + "$ref": "#/components/schemas/AgentProvider" + }, + { + "type": "null" + } + ] + }, + "security": { + "anyOf": [ + { + "items": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Security" + }, + "securitySchemes": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "$ref": "#/components/schemas/APIKeySecurityScheme" + }, + { + "$ref": "#/components/schemas/HTTPAuthSecurityScheme" + }, + { + "$ref": "#/components/schemas/OAuth2SecurityScheme" + }, + { + "$ref": "#/components/schemas/OpenIdConnectSecurityScheme" + }, + { + "$ref": "#/components/schemas/MutualTLSSecurityScheme" + } + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Securityschemes" + }, + "signatures": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/AgentCardSignature" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Signatures" + }, + "skills": { + "items": { + "$ref": "#/components/schemas/AgentSkill" + }, + "title": "Skills", + "type": "array" + }, + "supportsAuthenticatedExtendedCard": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Supportsauthenticatedextendedcard" + }, + "url": { + "title": "Url", + "type": "string" + }, + "version": { + "title": "Version", + "type": "string" + } + }, + "title": "AgentCard", + "type": "object" + }, + "AgentCardSignature": { + "description": "Represents a JWS signature of an AgentCard.", + "properties": { + "header": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Header" + }, + "protected": { + "title": "Protected", + "type": "string" + }, + "signature": { + "title": "Signature", + "type": "string" + } + }, + "title": "AgentCardSignature", + "type": "object" + }, + "AgentConfig": { + "properties": { + "agent_card_params": { + "$ref": "#/components/schemas/AgentCard" + }, + "agent_name": { + "title": "Agent Name", + "type": "string" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "litellm_params": { + "additionalProperties": true, + "title": "Litellm Params", + "type": "object" + }, + "object_permission": { + "$ref": "#/components/schemas/AgentObjectPermission" + }, + "rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Rpm Limit" + }, + "session_rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Rpm Limit" + }, + "session_tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Tpm Limit" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tpm Limit" + } + }, + "required": [ + "agent_name", + "agent_card_params" + ], + "title": "AgentConfig", + "type": "object" + }, + "AgentExtension": { + "description": "A declaration of a protocol extension supported by an Agent.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Params" + }, + "required": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Required" + }, + "uri": { + "title": "Uri", + "type": "string" + } + }, + "title": "AgentExtension", + "type": "object" + }, + "AgentInterface": { + "description": "Declares a combination of a target URL and a transport protocol.", + "properties": { + "transport": { + "title": "Transport", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "title": "AgentInterface", + "type": "object" + }, + "AgentMakePublicResponse": { + "properties": { + "message": { + "title": "Message", + "type": "string" + }, + "public_agent_groups": { + "items": { + "type": "string" + }, + "title": "Public Agent Groups", + "type": "array" + }, + "updated_by": { + "title": "Updated By", + "type": "string" + } + }, + "required": [ + "message", + "public_agent_groups", + "updated_by" + ], + "title": "AgentMakePublicResponse", + "type": "object" + }, + "AgentObjectPermission": { + "properties": { + "agents": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Agents" + }, + "mcp_access_groups": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Mcp Access Groups" + }, + "mcp_servers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Mcp Servers" + }, + "mcp_tool_permissions": { + "anyOf": [ + { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Tool Permissions" + }, + "models": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Models" + } + }, + "title": "AgentObjectPermission", + "type": "object" + }, + "AgentProvider": { + "description": "Represents the service provider of an agent.", + "properties": { + "organization": { + "title": "Organization", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "title": "AgentProvider", + "type": "object" + }, + "AgentResponse": { + "properties": { + "agent_card_params": { + "additionalProperties": true, + "title": "Agent Card Params", + "type": "object" + }, + "agent_id": { + "title": "Agent Id", + "type": "string" + }, + "agent_name": { + "title": "Agent Name", + "type": "string" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "litellm_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Litellm Params" + }, + "object_permission": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Object Permission" + }, + "rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Rpm Limit" + }, + "session_rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Rpm Limit" + }, + "session_tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Tpm Limit" + }, + "spend": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Spend" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tpm Limit" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated By" + } + }, + "required": [ + "agent_id", + "agent_name", + "agent_card_params" + ], + "title": "AgentResponse", + "type": "object" + }, + "AgentSkill": { + "description": "Represents a distinct capability or function that an agent can perform.", + "properties": { + "description": { + "title": "Description", + "type": "string" + }, + "examples": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Examples" + }, + "id": { + "title": "Id", + "type": "string" + }, + "inputModes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Inputmodes" + }, + "name": { + "title": "Name", + "type": "string" + }, + "outputModes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Outputmodes" + }, + "security": { + "anyOf": [ + { + "items": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Security" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + } + }, + "title": "AgentSkill", + "type": "object" + }, + "BreakdownMetrics": { + "description": "Breakdown of spend by different dimensions", + "properties": { + "api_keys": { + "additionalProperties": { + "$ref": "#/components/schemas/KeyMetricWithMetadata" + }, + "title": "Api Keys", + "type": "object" + }, + "endpoints": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Endpoints", + "type": "object" + }, + "entities": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Entities", + "type": "object" + }, + "mcp_servers": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Mcp Servers", + "type": "object" + }, + "model_groups": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Model Groups", + "type": "object" + }, + "models": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Models", + "type": "object" + }, + "providers": { + "additionalProperties": { + "$ref": "#/components/schemas/MetricWithMetadata" + }, + "title": "Providers", + "type": "object" + } + }, + "title": "BreakdownMetrics", + "type": "object" + }, + "DailySpendData": { + "properties": { + "breakdown": { + "$ref": "#/components/schemas/BreakdownMetrics" + }, + "date": { + "format": "date", + "title": "Date", + "type": "string" + }, + "metrics": { + "$ref": "#/components/schemas/SpendMetrics" + } + }, + "required": [ + "date", + "metrics" + ], + "title": "DailySpendData", + "type": "object" + }, + "DailySpendMetadata": { + "properties": { + "has_more": { + "default": false, + "title": "Has More", + "type": "boolean" + }, + "page": { + "default": 1, + "title": "Page", + "type": "integer" + }, + "total_api_requests": { + "default": 0, + "title": "Total Api Requests", + "type": "integer" + }, + "total_cache_creation_input_tokens": { + "default": 0, + "title": "Total Cache Creation Input Tokens", + "type": "integer" + }, + "total_cache_read_input_tokens": { + "default": 0, + "title": "Total Cache Read Input Tokens", + "type": "integer" + }, + "total_completion_tokens": { + "default": 0, + "title": "Total Completion Tokens", + "type": "integer" + }, + "total_failed_requests": { + "default": 0, + "title": "Total Failed Requests", + "type": "integer" + }, + "total_pages": { + "default": 1, + "title": "Total Pages", + "type": "integer" + }, + "total_prompt_tokens": { + "default": 0, + "title": "Total Prompt Tokens", + "type": "integer" + }, + "total_spend": { + "default": 0.0, + "title": "Total Spend", + "type": "number" + }, + "total_successful_requests": { + "default": 0, + "title": "Total Successful Requests", + "type": "integer" + }, + "total_tokens": { + "default": 0, + "title": "Total Tokens", + "type": "integer" + } + }, + "title": "DailySpendMetadata", + "type": "object" + }, + "HTTPAuthSecurityScheme": { + "description": "Defines a security scheme using HTTP authentication.", + "properties": { + "bearerFormat": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Bearerformat" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "scheme": { + "title": "Scheme", + "type": "string" + }, + "type": { + "const": "http", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "scheme", + "bearerFormat" + ], + "title": "HTTPAuthSecurityScheme", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "KeyMetadata": { + "description": "Metadata for a key", + "properties": { + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Alias" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + } + }, + "title": "KeyMetadata", + "type": "object" + }, + "KeyMetricWithMetadata": { + "description": "Base class for metrics with additional metadata", + "properties": { + "metadata": { + "$ref": "#/components/schemas/KeyMetadata" + }, + "metrics": { + "$ref": "#/components/schemas/SpendMetrics" + } + }, + "required": [ + "metrics" + ], + "title": "KeyMetricWithMetadata", + "type": "object" + }, + "MakeAgentsPublicRequest": { + "properties": { + "agent_ids": { + "items": { + "type": "string" + }, + "title": "Agent Ids", + "type": "array" + } + }, + "required": [ + "agent_ids" + ], + "title": "MakeAgentsPublicRequest", + "type": "object" + }, + "MetricWithMetadata": { + "properties": { + "api_key_breakdown": { + "additionalProperties": { + "$ref": "#/components/schemas/KeyMetricWithMetadata" + }, + "title": "Api Key Breakdown", + "type": "object" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + }, + "metrics": { + "$ref": "#/components/schemas/SpendMetrics" + } + }, + "required": [ + "metrics" + ], + "title": "MetricWithMetadata", + "type": "object" + }, + "MutualTLSSecurityScheme": { + "description": "Defines a security scheme using mTLS authentication.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "type": { + "const": "mutualTLS", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "MutualTLSSecurityScheme", + "type": "object" + }, + "OAuth2SecurityScheme": { + "description": "Defines a security scheme using OAuth 2.0.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "flows": { + "$ref": "#/components/schemas/OAuthFlows" + }, + "oauth2MetadataUrl": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Oauth2Metadataurl" + }, + "type": { + "const": "oauth2", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "flows", + "oauth2MetadataUrl" + ], + "title": "OAuth2SecurityScheme", + "type": "object" + }, + "OAuthFlows": { + "description": "Defines the configuration for the supported OAuth 2.0 flows.", + "properties": { + "authorizationCode": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Authorizationcode" + }, + "clientCredentials": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Clientcredentials" + }, + "implicit": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Implicit" + }, + "password": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Password" + } + }, + "title": "OAuthFlows", + "type": "object" + }, + "OpenIdConnectSecurityScheme": { + "description": "Defines a security scheme using OpenID Connect.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "openIdConnectUrl": { + "title": "Openidconnecturl", + "type": "string" + }, + "type": { + "const": "openIdConnect", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "openIdConnectUrl" + ], + "title": "OpenIdConnectSecurityScheme", + "type": "object" + }, + "PatchAgentRequest": { + "properties": { + "agent_card_params": { + "$ref": "#/components/schemas/AgentCard" + }, + "agent_name": { + "title": "Agent Name", + "type": "string" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "litellm_params": { + "additionalProperties": true, + "title": "Litellm Params", + "type": "object" + }, + "object_permission": { + "$ref": "#/components/schemas/AgentObjectPermission" + }, + "rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Rpm Limit" + }, + "session_rpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Rpm Limit" + }, + "session_tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Session Tpm Limit" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "tpm_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tpm Limit" + } + }, + "title": "PatchAgentRequest", + "type": "object" + }, + "SpendAnalyticsPaginatedResponse": { + "properties": { + "metadata": { + "$ref": "#/components/schemas/DailySpendMetadata" + }, + "results": { + "items": { + "$ref": "#/components/schemas/DailySpendData" + }, + "title": "Results", + "type": "array" + } + }, + "required": [ + "results" + ], + "title": "SpendAnalyticsPaginatedResponse", + "type": "object" + }, + "SpendMetrics": { + "properties": { + "api_requests": { + "default": 0, + "title": "Api Requests", + "type": "integer" + }, + "cache_creation_input_tokens": { + "default": 0, + "title": "Cache Creation Input Tokens", + "type": "integer" + }, + "cache_read_input_tokens": { + "default": 0, + "title": "Cache Read Input Tokens", + "type": "integer" + }, + "completion_tokens": { + "default": 0, + "title": "Completion Tokens", + "type": "integer" + }, + "failed_requests": { + "default": 0, + "title": "Failed Requests", + "type": "integer" + }, + "prompt_tokens": { + "default": 0, + "title": "Prompt Tokens", + "type": "integer" + }, + "spend": { + "default": 0.0, + "title": "Spend", + "type": "number" + }, + "successful_requests": { + "default": 0, + "title": "Successful Requests", + "type": "integer" + }, + "total_tokens": { + "default": 0, + "title": "Total Tokens", + "type": "integer" + } + }, + "title": "SpendMetrics", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/agent/daily/activity": { + "get": { + "description": "Get daily activity for specific agents or all accessible agents.", + "operationId": "get_agent_daily_activity_agent_daily_activity_get", + "parameters": [ + { + "in": "query", + "name": "agent_ids", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Agent Ids" + } + }, + { + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Start Date" + } + }, + { + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "End Date" + } + }, + { + "in": "query", + "name": "model", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + } + }, + { + "in": "query", + "name": "api_key", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Key" + } + }, + { + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "title": "Page", + "type": "integer" + } + }, + { + "in": "query", + "name": "page_size", + "required": false, + "schema": { + "default": 10, + "title": "Page Size", + "type": "integer" + } + }, + { + "in": "query", + "name": "exclude_agent_ids", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Exclude Agent Ids" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SpendAnalyticsPaginatedResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Agent Daily Activity", + "tags": [ + "agents" + ] + } + }, + "/v1/agents": { + "get": { + "description": "Example usage:\n```\ncurl -X GET \"http://localhost:4000/v1/agents\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nPass `?health_check=true` to filter out agents whose URL is unreachable:\n```\ncurl -X GET \"http://localhost:4000/v1/agents?health_check=true\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nReturns: List[AgentResponse]", + "operationId": "get_agents_v1_agents_get", + "parameters": [ + { + "description": "When true, performs a GET request to each agent's URL. Agents with reachable URLs (HTTP status < 500) and agents without a URL are returned; unreachable agents are filtered out.", + "in": "query", + "name": "health_check", + "required": false, + "schema": { + "default": false, + "description": "When true, performs a GET request to each agent's URL. Agents with reachable URLs (HTTP status < 500) and agents without a URL are returned; unreachable agents are filtered out.", + "title": "Health Check", + "type": "boolean" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/AgentResponse" + }, + "title": "Response Get Agents V1 Agents Get", + "type": "array" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Agents", + "tags": [ + "agents" + ] + }, + "post": { + "description": "Create a new agent\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/agents\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": {\n \"agent_name\": \"my-custom-agent\",\n \"agent_card_params\": {\n \"protocolVersion\": \"1.0\",\n \"name\": \"Hello World Agent\",\n \"description\": \"Just a hello world agent\",\n \"url\": \"http://localhost:9999/\",\n \"version\": \"1.0.0\",\n \"defaultInputModes\": [\"text\"],\n \"defaultOutputModes\": [\"text\"],\n \"capabilities\": {\n \"streaming\": true\n },\n \"skills\": [\n {\n \"id\": \"hello_world\",\n \"name\": \"Returns hello world\",\n \"description\": \"just returns hello world\",\n \"tags\": [\"hello world\"],\n \"examples\": [\"hi\", \"hello world\"]\n }\n ]\n },\n \"litellm_params\": {\n \"make_public\": true\n }\n }\n }'\n```", + "operationId": "create_agent_v1_agents_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentConfig" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Agent", + "tags": [ + "agents" + ] + } + }, + "/v1/agents/make_public": { + "post": { + "description": "Make multiple agents publicly discoverable\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/v1/agents/make_public\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent_ids\": [\"123e4567-e89b-12d3-a456-426614174000\", \"123e4567-e89b-12d3-a456-426614174001\"]\n }'\n```\n\nExample Response:\n```json\n{\n \"agent_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"agent_name\": \"my-custom-agent\",\n \"litellm_params\": {\n \"make_public\": true\n },\n \"agent_card_params\": {...},\n \"created_at\": \"2025-11-15T10:30:00Z\",\n \"updated_at\": \"2025-11-15T10:35:00Z\",\n \"created_by\": \"user123\",\n \"updated_by\": \"user123\"\n}\n```", + "operationId": "make_agents_public_v1_agents_make_public_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MakeAgentsPublicRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMakePublicResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Make Agents Public", + "tags": [ + "agents" + ] + } + }, + "/v1/agents/{agent_id}": { + "delete": { + "description": "Delete an agent\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Agent 123e4567-e89b-12d3-a456-426614174000 deleted successfully\"\n}\n```", + "operationId": "delete_agent_v1_agents__agent_id__delete", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Agent", + "tags": [ + "agents" + ] + }, + "get": { + "description": "Get a specific agent by ID\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```", + "operationId": "get_agent_by_id_v1_agents__agent_id__get", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Agent By Id", + "tags": [ + "agents" + ] + }, + "patch": { + "description": "Update an existing agent\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": {\n \"agent_name\": \"updated-agent\",\n \"agent_card_params\": {\n \"protocolVersion\": \"1.0\",\n \"name\": \"Updated Agent\",\n \"description\": \"Updated description\",\n \"url\": \"http://localhost:9999/\",\n \"version\": \"1.1.0\",\n \"defaultInputModes\": [\"text\"],\n \"defaultOutputModes\": [\"text\"],\n \"capabilities\": {\n \"streaming\": true\n },\n \"skills\": []\n },\n \"litellm_params\": {\n \"make_public\": false\n }\n }\n }'\n```", + "operationId": "patch_agent_v1_agents__agent_id__patch", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PatchAgentRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Patch Agent", + "tags": [ + "agents" + ] + }, + "put": { + "description": "Update an existing agent\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/agents/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": {\n \"agent_name\": \"updated-agent\",\n \"agent_card_params\": {\n \"protocolVersion\": \"1.0\",\n \"name\": \"Updated Agent\",\n \"description\": \"Updated description\",\n \"url\": \"http://localhost:9999/\",\n \"version\": \"1.1.0\",\n \"defaultInputModes\": [\"text\"],\n \"defaultOutputModes\": [\"text\"],\n \"capabilities\": {\n \"streaming\": true\n },\n \"skills\": []\n },\n \"litellm_params\": {\n \"make_public\": false\n }\n }\n }'\n```", + "operationId": "update_agent_v1_agents__agent_id__put", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentConfig" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Agent", + "tags": [ + "agents" + ] + } + }, + "/v1/agents/{agent_id}/make_public": { + "post": { + "description": "Make an agent publicly discoverable\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/v1/agents/123e4567-e89b-12d3-a456-426614174000/make_public\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\"\n```\n\nExample Response:\n```json\n{\n \"agent_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"agent_name\": \"my-custom-agent\",\n \"litellm_params\": {\n \"make_public\": true\n },\n \"agent_card_params\": {...},\n \"created_at\": \"2025-11-15T10:30:00Z\",\n \"updated_at\": \"2025-11-15T10:35:00Z\",\n \"created_by\": \"user123\",\n \"updated_by\": \"user123\"\n}\n```", + "operationId": "make_agent_public_v1_agents__agent_id__make_public_post", + "parameters": [ + { + "in": "path", + "name": "agent_id", + "required": true, + "schema": { + "title": "Agent Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMakePublicResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Make Agent Public", + "tags": [ + "agents" + ] + } + } + } + }, + "anthropic_passthrough": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/anthropic/{endpoint}": { + "delete": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", + "operationId": "anthropic_proxy_route_anthropic__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Proxy Route", + "tags": [ + "anthropic_passthrough" + ] + }, + "get": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", + "operationId": "anthropic_proxy_route_anthropic__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Proxy Route", + "tags": [ + "anthropic_passthrough" + ] + }, + "patch": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", + "operationId": "anthropic_proxy_route_anthropic__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Proxy Route", + "tags": [ + "anthropic_passthrough" + ] + }, + "post": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", + "operationId": "anthropic_proxy_route_anthropic__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Proxy Route", + "tags": [ + "anthropic_passthrough" + ] + }, + "put": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)", + "operationId": "anthropic_proxy_route_anthropic__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Proxy Route", + "tags": [ + "anthropic_passthrough" + ] + } + }, + "/api/event_logging/batch": { + "post": { + "description": "Stubbed endpoint for Anthropic event logging batch requests.\n\nThis endpoint accepts event logging requests but does nothing with them.\nIt exists to prevent 404 errors from Claude Code clients that send telemetry.", + "operationId": "event_logging_batch_api_event_logging_batch_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Event Logging Batch", + "tags": [ + "anthropic_passthrough" + ] + } + }, + "/v1/messages": { + "post": { + "description": "Use `{PROXY_BASE_URL}/anthropic/v1/messages` instead - [Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion).\n\nThis was a BETA endpoint that calls 100+ LLMs in the anthropic format.", + "operationId": "anthropic_response_v1_messages_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Anthropic Response", + "tags": [ + "anthropic_passthrough" + ] + } + }, + "/v1/messages/count_tokens": { + "post": { + "description": "Count tokens for Anthropic Messages API format.\n\nThis endpoint follows the Anthropic Messages API token counting specification.\nIt accepts the same parameters as the /v1/messages endpoint but returns\ntoken counts instead of generating a response.\n\nExample usage:\n```\ncurl -X POST \"http://localhost:4000/v1/messages/count_tokens?beta=true\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" -d '{\n \"model\": \"claude-3-sonnet-20240229\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Hello Claude!\"}]\n }'\n```\n\nReturns: {\"input_tokens\": }", + "operationId": "count_tokens_v1_messages_count_tokens_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Count Tokens", + "tags": [ + "anthropic_passthrough" + ] + } + } + } + }, + "anthropic_skills": { + "components": { + "schemas": { + "DeleteSkillResponse": { + "description": "Response from deleting a skill", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "type": { + "default": "skill_deleted", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id" + ], + "title": "DeleteSkillResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListSkillsResponse": { + "description": "Response from listing skills", + "properties": { + "data": { + "items": { + "$ref": "#/components/schemas/Skill" + }, + "title": "Data", + "type": "array" + }, + "has_more": { + "default": false, + "title": "Has More", + "type": "boolean" + }, + "next_page": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next Page" + } + }, + "required": [ + "data" + ], + "title": "ListSkillsResponse", + "type": "object" + }, + "Skill": { + "description": "Represents a skill from the Anthropic Skills API", + "properties": { + "created_at": { + "title": "Created At", + "type": "string" + }, + "display_title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Display Title" + }, + "id": { + "title": "Id", + "type": "string" + }, + "latest_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Latest Version" + }, + "source": { + "title": "Source", + "type": "string" + }, + "type": { + "default": "skill", + "title": "Type", + "type": "string" + }, + "updated_at": { + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "id", + "created_at", + "source", + "updated_at" + ], + "title": "Skill", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/v1/skills": { + "get": { + "description": "List skills on Anthropic.\n\nRequires `?beta=true` query parameter.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: claude-account-1`\n- Pass model via query: `?model=claude-account-1`\n- Pass model via body: `{\"model\": \"claude-account-1\"}`\n\nExample usage:\n```bash\n# Basic usage\ncurl \"http://localhost:4000/v1/skills?beta=true&limit=10\" -H \"Authorization: Bearer your-key\"\n\n# With model-based routing\ncurl \"http://localhost:4000/v1/skills?beta=true&limit=10\" -H \"Authorization: Bearer your-key\" -H \"x-litellm-model: claude-account-1\"\n```\n\nReturns: ListSkillsResponse with list of skills", + "operationId": "list_skills_v1_skills_get", + "parameters": [ + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 10, + "title": "Limit" + } + }, + { + "in": "query", + "name": "after_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After Id" + } + }, + { + "in": "query", + "name": "before_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before Id" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "anthropic", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListSkillsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Skills", + "tags": [ + "anthropic_skills" + ] + }, + "post": { + "description": "Create a new skill on Anthropic.\n\nRequires `?beta=true` query parameter.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: claude-account-1`\n- Pass model via query: `?model=claude-account-1`\n- Pass model via form field: `model=claude-account-1`\n\nExample usage:\n```bash\n# Basic usage\ncurl -X POST \"http://localhost:4000/v1/skills?beta=true\" -H \"Content-Type: multipart/form-data\" -H \"Authorization: Bearer your-key\" -F \"display_title=My Skill\" -F \"files[]=@skill.zip\"\n\n# With model-based routing\ncurl -X POST \"http://localhost:4000/v1/skills?beta=true\" -H \"Content-Type: multipart/form-data\" -H \"Authorization: Bearer your-key\" -H \"x-litellm-model: claude-account-1\" -F \"display_title=My Skill\" -F \"files[]=@skill.zip\"\n```\n\nReturns: Skill object with id, display_title, etc.", + "operationId": "create_skill_v1_skills_post", + "parameters": [ + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "anthropic", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Skill" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Skill", + "tags": [ + "anthropic_skills" + ] + } + }, + "/v1/skills/{skill_id}": { + "delete": { + "description": "Delete a skill by ID from Anthropic.\n\nRequires `?beta=true` query parameter.\n\nNote: Anthropic does not allow deleting skills with existing versions.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: claude-account-1`\n- Pass model via query: `?model=claude-account-1`\n- Pass model via body: `{\"model\": \"claude-account-1\"}`\n\nExample usage:\n```bash\n# Basic usage\ncurl -X DELETE \"http://localhost:4000/v1/skills/skill_123?beta=true\" -H \"Authorization: Bearer your-key\"\n\n# With model-based routing\ncurl -X DELETE \"http://localhost:4000/v1/skills/skill_123?beta=true\" -H \"Authorization: Bearer your-key\" -H \"x-litellm-model: claude-account-1\"\n```\n\nReturns: DeleteSkillResponse with type=\"skill_deleted\"", + "operationId": "delete_skill_v1_skills__skill_id__delete", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "anthropic", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteSkillResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Skill", + "tags": [ + "anthropic_skills" + ] + }, + "get": { + "description": "Get a specific skill by ID from Anthropic.\n\nRequires `?beta=true` query parameter.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: claude-account-1`\n- Pass model via query: `?model=claude-account-1`\n- Pass model via body: `{\"model\": \"claude-account-1\"}`\n\nExample usage:\n```bash\n# Basic usage\ncurl \"http://localhost:4000/v1/skills/skill_123?beta=true\" -H \"Authorization: Bearer your-key\"\n\n# With model-based routing\ncurl \"http://localhost:4000/v1/skills/skill_123?beta=true\" -H \"Authorization: Bearer your-key\" -H \"x-litellm-model: claude-account-1\"\n```\n\nReturns: Skill object", + "operationId": "get_skill_v1_skills__skill_id__get", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "anthropic", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Skill" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Skill", + "tags": [ + "anthropic_skills" + ] + } + } + } + }, + "claude_code_marketplace": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListPluginsResponse": { + "description": "Response from listing plugins.", + "properties": { + "count": { + "title": "Count", + "type": "integer" + }, + "plugins": { + "items": { + "$ref": "#/components/schemas/PluginListItem" + }, + "title": "Plugins", + "type": "array" + } + }, + "required": [ + "plugins", + "count" + ], + "title": "ListPluginsResponse", + "type": "object" + }, + "PluginAuthor": { + "description": "Plugin author information.", + "properties": { + "email": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Author email", + "title": "Email" + }, + "name": { + "description": "Author name", + "title": "Name", + "type": "string" + } + }, + "required": [ + "name" + ], + "title": "PluginAuthor", + "type": "object" + }, + "PluginListItem": { + "description": "Plugin item in list responses.", + "properties": { + "author": { + "anyOf": [ + { + "$ref": "#/components/schemas/PluginAuthor" + }, + { + "type": "null" + } + ] + }, + "category": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Category" + }, + "created_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "domain": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Domain" + }, + "enabled": { + "title": "Enabled", + "type": "boolean" + }, + "homepage": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Homepage" + }, + "id": { + "title": "Id", + "type": "string" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "name": { + "title": "Name", + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace" + }, + "source": { + "additionalProperties": { + "type": "string" + }, + "title": "Source", + "type": "object" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "id", + "name", + "version", + "description", + "source", + "enabled", + "created_at", + "updated_at" + ], + "title": "PluginListItem", + "type": "object" + }, + "RegisterPluginRequest": { + "description": "Request body for registering a plugin in the marketplace.\n\nLiteLLM acts as a registry/discovery layer. Plugins are hosted on\nGitHub/GitLab/Bitbucket and referenced by their git source.", + "properties": { + "author": { + "anyOf": [ + { + "$ref": "#/components/schemas/PluginAuthor" + }, + { + "type": "null" + } + ], + "description": "Plugin author" + }, + "category": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Plugin category", + "title": "Category" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Plugin description", + "title": "Description" + }, + "domain": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Skill domain (e.g., 'Productivity')", + "title": "Domain" + }, + "homepage": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Plugin homepage URL", + "title": "Homepage" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Search keywords", + "title": "Keywords" + }, + "name": { + "description": "Plugin name (kebab-case, e.g., 'my-plugin')", + "pattern": "^[a-z0-9-]+$", + "title": "Name", + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Skill namespace within domain (e.g., 'workflows')", + "title": "Namespace" + }, + "source": { + "additionalProperties": { + "type": "string" + }, + "description": "Git source reference. Supported formats:\n- GitHub: {'source': 'github', 'repo': 'org/repo'}\n- Git URL: {'source': 'url', 'url': 'https://github.com/org/repo.git'}\n- Git Subdir: {'source': 'git-subdir', 'url': 'https://github.com/org/repo.git', 'path': 'plugins/plugin-name'}", + "title": "Source", + "type": "object" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "1.0.0", + "description": "Semantic version", + "title": "Version" + } + }, + "required": [ + "name", + "source" + ], + "title": "RegisterPluginRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/claude-code/marketplace.json": { + "get": { + "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin install my-plugin@litellm\n ```", + "operationId": "get_marketplace_claude_code_marketplace_json_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Get Marketplace", + "tags": [ + "claude_code_marketplace" + ] + } + }, + "/claude-code/plugins": { + "get": { + "description": "List all plugins in the marketplace.\n\nParameters:\n - enabled_only: If true, only return enabled plugins\n\nReturns:\n List of plugins with their metadata.", + "operationId": "list_plugins_claude_code_plugins_get", + "parameters": [ + { + "in": "query", + "name": "enabled_only", + "required": false, + "schema": { + "default": false, + "title": "Enabled Only", + "type": "boolean" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListPluginsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Plugins", + "tags": [ + "claude_code_marketplace" + ] + }, + "post": { + "description": "Register a plugin in the LiteLLM marketplace.\n\nLiteLLM acts as a registry/discovery layer. Plugins are hosted on\nGitHub/GitLab/Bitbucket. Claude Code will clone from the git source\nwhen users install.\n\nParameters:\n - name: Plugin name (kebab-case)\n - source: Git source reference (github, url, or git-subdir format)\n - version: Semantic version (optional)\n - description: Plugin description (optional)\n - author: Author information (optional)\n - homepage: Plugin homepage URL (optional)\n - keywords: Search keywords (optional)\n - category: Plugin category (optional)\n\nReturns:\n Registration status and plugin information.\n\nExample:\n ```bash\n curl -X POST http://localhost:4000/claude-code/plugins \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"name\": \"my-plugin\",\n \"source\": {\"source\": \"github\", \"repo\": \"org/my-plugin\"},\n \"version\": \"1.0.0\",\n \"description\": \"My awesome plugin\"\n }'\n ```", + "operationId": "register_plugin_claude_code_plugins_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterPluginRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Register Plugin", + "tags": [ + "claude_code_marketplace" + ] + } + }, + "/claude-code/plugins/{plugin_name}": { + "delete": { + "description": "Delete a plugin from the marketplace.\n\nParameters:\n - plugin_name: The name of the plugin to delete", + "operationId": "delete_plugin_claude_code_plugins__plugin_name__delete", + "parameters": [ + { + "in": "path", + "name": "plugin_name", + "required": true, + "schema": { + "title": "Plugin Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Plugin", + "tags": [ + "claude_code_marketplace" + ] + }, + "get": { + "description": "Get details of a specific plugin.\n\nParameters:\n - plugin_name: The name of the plugin\n\nReturns:\n Plugin details including source and metadata.", + "operationId": "get_plugin_claude_code_plugins__plugin_name__get", + "parameters": [ + { + "in": "path", + "name": "plugin_name", + "required": true, + "schema": { + "title": "Plugin Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Plugin", + "tags": [ + "claude_code_marketplace" + ] + } + }, + "/claude-code/plugins/{plugin_name}/disable": { + "post": { + "description": "Disable a plugin without deleting it.\n\nParameters:\n - plugin_name: The name of the plugin to disable", + "operationId": "disable_plugin_claude_code_plugins__plugin_name__disable_post", + "parameters": [ + { + "in": "path", + "name": "plugin_name", + "required": true, + "schema": { + "title": "Plugin Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Disable Plugin", + "tags": [ + "claude_code_marketplace" + ] + } + }, + "/claude-code/plugins/{plugin_name}/enable": { + "post": { + "description": "Enable a disabled plugin.\n\nParameters:\n - plugin_name: The name of the plugin to enable", + "operationId": "enable_plugin_claude_code_plugins__plugin_name__enable_post", + "parameters": [ + { + "in": "path", + "name": "plugin_name", + "required": true, + "schema": { + "title": "Plugin Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Enable Plugin", + "tags": [ + "claude_code_marketplace" + ] + } + } + } + }, + "cloudzero": { + "components": { + "schemas": { + "CloudZeroExportRequest": { + "description": "Request model for CloudZero export operations", + "properties": { + "end_time_utc": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "End time for data export in UTC", + "title": "End Time Utc" + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Optional limit on number of records to export", + "title": "Limit" + }, + "operation": { + "default": "replace_hourly", + "description": "CloudZero operation type (replace_hourly or sum)", + "title": "Operation", + "type": "string" + }, + "start_time_utc": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Start time for data export in UTC", + "title": "Start Time Utc" + } + }, + "title": "CloudZeroExportRequest", + "type": "object" + }, + "CloudZeroExportResponse": { + "description": "Response model for CloudZero export operations", + "properties": { + "dry_run_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Dry run data including usage data and CBF transformed data", + "title": "Dry Run Data" + }, + "message": { + "title": "Message", + "type": "string" + }, + "records_exported": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Records Exported" + }, + "status": { + "title": "Status", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Summary statistics for dry run", + "title": "Summary" + } + }, + "required": [ + "message", + "status" + ], + "title": "CloudZeroExportResponse", + "type": "object" + }, + "CloudZeroInitRequest": { + "description": "Request model for initializing CloudZero settings", + "properties": { + "api_key": { + "description": "CloudZero API key for authentication", + "title": "Api Key", + "type": "string" + }, + "connection_id": { + "description": "CloudZero connection ID for data submission", + "title": "Connection Id", + "type": "string" + }, + "timezone": { + "default": "UTC", + "description": "Timezone for date handling (default: UTC)", + "title": "Timezone", + "type": "string" + } + }, + "required": [ + "api_key", + "connection_id" + ], + "title": "CloudZeroInitRequest", + "type": "object" + }, + "CloudZeroInitResponse": { + "description": "Response model for CloudZero initialization", + "properties": { + "message": { + "title": "Message", + "type": "string" + }, + "status": { + "title": "Status", + "type": "string" + } + }, + "required": [ + "message", + "status" + ], + "title": "CloudZeroInitResponse", + "type": "object" + }, + "CloudZeroSettingsUpdate": { + "description": "Request model for updating CloudZero settings", + "properties": { + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New CloudZero API key for authentication", + "title": "Api Key" + }, + "connection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New CloudZero connection ID for data submission", + "title": "Connection Id" + }, + "timezone": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New timezone for date handling", + "title": "Timezone" + } + }, + "title": "CloudZeroSettingsUpdate", + "type": "object" + }, + "CloudZeroSettingsView": { + "description": "Response model for viewing CloudZero settings with masked API key", + "properties": { + "api_key_masked": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Masked API key showing only first 4 and last 4 characters", + "title": "Api Key Masked" + }, + "connection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "CloudZero connection ID for data submission", + "title": "Connection Id" + }, + "status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Configuration status", + "title": "Status" + }, + "timezone": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Timezone for date handling", + "title": "Timezone" + } + }, + "title": "CloudZeroSettingsView", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/cloudzero/delete": { + "delete": { + "description": "Delete CloudZero settings from the database.\n\nThis endpoint removes the CloudZero configuration (API key, connection ID, timezone)\nfrom the proxy database. Only the CloudZero settings entry will be deleted;\nother configuration values in the database will remain unchanged.\n\nOnly admin users can delete CloudZero settings.", + "operationId": "delete_cloudzero_settings_cloudzero_delete_delete", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroInitResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Cloudzero Settings", + "tags": [ + "cloudzero" + ] + } + }, + "/cloudzero/dry-run": { + "post": { + "description": "Perform a dry run export using the CloudZero logger.\n\nThis endpoint uses the CloudZero logger to perform a dry run export,\nwhich returns the data that would be exported without actually sending it to CloudZero.\n\nParameters:\n- limit: Optional limit on number of records to process (default: 10000)\n\nReturns:\n- usage_data: Sample of the raw usage data (first 50 records)\n- cbf_data: CloudZero CBF formatted data ready for export\n- summary: Statistics including total cost, tokens, and record counts\n\nOnly admin users can perform CloudZero exports.", + "operationId": "cloudzero_dry_run_export_cloudzero_dry_run_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroExportRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroExportResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Cloudzero Dry Run Export", + "tags": [ + "cloudzero" + ] + } + }, + "/cloudzero/export": { + "post": { + "description": "Perform an actual export using the CloudZero logger.\n\nThis endpoint uses the CloudZero logger to export usage data to CloudZero AnyCost API.\n\nParameters:\n- limit: Optional limit on number of records to export\n- operation: CloudZero operation type (\"replace_hourly\" or \"sum\", default: \"replace_hourly\")\n\nOnly admin users can perform CloudZero exports.", + "operationId": "cloudzero_export_cloudzero_export_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroExportRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroExportResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Cloudzero Export", + "tags": [ + "cloudzero" + ] + } + }, + "/cloudzero/init": { + "post": { + "description": "Initialize CloudZero settings and store in the database.\n\nThis endpoint stores the CloudZero API key, connection ID, and timezone configuration\nin the proxy database for use by the CloudZero logger.\n\nParameters:\n- api_key: CloudZero API key for authentication\n- connection_id: CloudZero connection ID for data submission\n- timezone: Timezone for date handling (default: UTC)\n\nOnly admin users can configure CloudZero settings.", + "operationId": "init_cloudzero_settings_cloudzero_init_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroInitRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroInitResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Init Cloudzero Settings", + "tags": [ + "cloudzero" + ] + } + }, + "/cloudzero/settings": { + "get": { + "description": "View current CloudZero settings.\n\nReturns the current CloudZero configuration with the API key masked for security.\nOnly the first 4 and last 4 characters of the API key are shown.\nReturns null/empty values when settings are not configured (consistent with other settings endpoints).\n\nOnly admin users can view CloudZero settings.", + "operationId": "get_cloudzero_settings_cloudzero_settings_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroSettingsView" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Cloudzero Settings", + "tags": [ + "cloudzero" + ] + }, + "put": { + "description": "Update existing CloudZero settings.\n\nAllows updating individual CloudZero configuration fields without requiring all fields.\nOnly provided fields will be updated; others will remain unchanged.\n\nParameters:\n- api_key: (Optional) New CloudZero API key for authentication\n- connection_id: (Optional) New CloudZero connection ID for data submission\n- timezone: (Optional) New timezone for date handling\n\nOnly admin users can update CloudZero settings.", + "operationId": "update_cloudzero_settings_cloudzero_settings_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroSettingsUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CloudZeroInitResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Cloudzero Settings", + "tags": [ + "cloudzero" + ] + } + } + } + }, + "compliance": { + "components": { + "schemas": { + "ComplianceCheckRequest": { + "description": "Request payload for compliance check endpoints.\n\nMirrors the spend log fields needed for compliance evaluation.", + "properties": { + "guardrail_information": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Guardrail Information" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "timestamp": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Timestamp" + }, + "user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Id" + } + }, + "required": [ + "request_id" + ], + "title": "ComplianceCheckRequest", + "type": "object" + }, + "ComplianceCheckResult": { + "description": "Result of a single compliance check.", + "properties": { + "article": { + "title": "Article", + "type": "string" + }, + "check_name": { + "title": "Check Name", + "type": "string" + }, + "detail": { + "title": "Detail", + "type": "string" + }, + "passed": { + "title": "Passed", + "type": "boolean" + } + }, + "required": [ + "check_name", + "article", + "passed", + "detail" + ], + "title": "ComplianceCheckResult", + "type": "object" + }, + "ComplianceResponse": { + "description": "Response from a compliance check endpoint.", + "properties": { + "checks": { + "items": { + "$ref": "#/components/schemas/ComplianceCheckResult" + }, + "title": "Checks", + "type": "array" + }, + "compliant": { + "title": "Compliant", + "type": "boolean" + }, + "regulation": { + "title": "Regulation", + "type": "string" + } + }, + "required": [ + "compliant", + "regulation", + "checks" + ], + "title": "ComplianceResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/compliance/eu-ai-act": { + "post": { + "description": "Check EU AI Act compliance for a spend log entry.\n\nChecks:\n- Art. 9: Guardrails applied (any guardrail)\n- Art. 5: Content screened before LLM (pre-call guardrails)\n- Art. 12: Audit record complete (user_id, model, timestamp, guardrail_results)", + "operationId": "check_eu_ai_act_compliance_compliance_eu_ai_act_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ComplianceCheckRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ComplianceResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Check Eu Ai Act Compliance", + "tags": [ + "compliance" + ] + } + }, + "/compliance/gdpr": { + "post": { + "description": "Check GDPR compliance for a spend log entry.\n\nChecks:\n- Art. 32: Data protection applied (pre-call guardrails)\n- Art. 5(1)(c): Sensitive data protected (masked/blocked or no issues)\n- Art. 30: Audit record complete (user_id, model, timestamp, guardrail_results)", + "operationId": "check_gdpr_compliance_compliance_gdpr_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ComplianceCheckRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ComplianceResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Check Gdpr Compliance", + "tags": [ + "compliance" + ] + } + } + } + }, + "config_overrides": { + "components": { + "schemas": { + "ConfigOverrideSettingsResponse": { + "description": "Response model for config override settings GET endpoints.", + "properties": { + "config_type": { + "description": "The type of config override", + "title": "Config Type", + "type": "string" + }, + "field_schema": { + "additionalProperties": true, + "description": "Schema information for UI rendering", + "title": "Field Schema", + "type": "object" + }, + "values": { + "additionalProperties": true, + "description": "Current configuration values (sensitive fields decrypted)", + "title": "Values", + "type": "object" + } + }, + "required": [ + "config_type", + "values", + "field_schema" + ], + "title": "ConfigOverrideSettingsResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "HashicorpVaultConfig": { + "description": "Configuration for Hashicorp Vault secret manager integration.", + "properties": { + "approle_mount_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Mount path for the AppRole auth method (default: approle)", + "title": "Approle Mount Path" + }, + "approle_role_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Role ID for Vault AppRole authentication", + "title": "Approle Role Id" + }, + "approle_secret_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Secret ID for Vault AppRole authentication", + "title": "Approle Secret Id" + }, + "client_cert": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to the client TLS certificate for Vault", + "title": "Client Cert" + }, + "client_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to the client TLS private key for Vault", + "title": "Client Key" + }, + "vault_addr": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The address of the Vault server (e.g., https://vault.example.com:8200)", + "title": "Vault Addr" + }, + "vault_cert_role": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Certificate role name for TLS cert authentication", + "title": "Vault Cert Role" + }, + "vault_mount_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "KV engine mount name (default: secret)", + "title": "Vault Mount Name" + }, + "vault_namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Vault namespace (for multi-tenant Vault, sent as X-Vault-Namespace header)", + "title": "Vault Namespace" + }, + "vault_path_prefix": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional path prefix for secrets (e.g., myapp -> secret/data/myapp/{secret_name})", + "title": "Vault Path Prefix" + }, + "vault_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Token for Vault token-based authentication", + "title": "Vault Token" + } + }, + "title": "HashicorpVaultConfig", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/config_overrides/hashicorp_vault": { + "delete": { + "description": "Delete Hashicorp Vault configuration. Idempotent.", + "operationId": "delete_hashicorp_vault_config_config_overrides_hashicorp_vault_delete", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Hashicorp Vault Config", + "tags": [ + "config_overrides" + ] + }, + "get": { + "description": "Get current Hashicorp Vault configuration.\nReturns decrypted values from DB, or falls back to current env vars.", + "operationId": "get_hashicorp_vault_config_config_overrides_hashicorp_vault_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ConfigOverrideSettingsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Hashicorp Vault Config", + "tags": [ + "config_overrides" + ] + }, + "post": { + "description": "Update Hashicorp Vault secret manager configuration.\nSets environment variables, encrypts sensitive fields, and stores in DB.\nReinitializes the secret manager on this pod.", + "operationId": "update_hashicorp_vault_config_config_overrides_hashicorp_vault_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HashicorpVaultConfig" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Hashicorp Vault Config", + "tags": [ + "config_overrides" + ] + } + }, + "/config_overrides/hashicorp_vault/test_connection": { + "post": { + "description": "Test the connection to the currently configured Hashicorp Vault.\nUses the already-initialized secret manager client. Does not modify any state.", + "operationId": "test_hashicorp_vault_connection_config_overrides_hashicorp_vault_test_connection_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Hashicorp Vault Connection", + "tags": [ + "config_overrides" + ] + } + } + } + }, + "evals": { + "components": { + "schemas": { + "CancelEvalResponse": { + "description": "Response from cancelling an evaluation", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "object": { + "default": "eval", + "title": "Object", + "type": "string" + }, + "status": { + "const": "cancelled", + "title": "Status", + "type": "string" + } + }, + "required": [ + "id", + "status" + ], + "title": "CancelEvalResponse", + "type": "object" + }, + "CancelRunResponse": { + "description": "Response from cancelling a run", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "object": { + "default": "eval.run", + "title": "Object", + "type": "string" + }, + "status": { + "const": "cancelled", + "title": "Status", + "type": "string" + } + }, + "required": [ + "id", + "status" + ], + "title": "CancelRunResponse", + "type": "object" + }, + "DeleteEvalResponse": { + "description": "Response from deleting an evaluation", + "properties": { + "deleted": { + "title": "Deleted", + "type": "boolean" + }, + "eval_id": { + "title": "Eval Id", + "type": "string" + }, + "object": { + "default": "eval.deleted", + "title": "Object", + "type": "string" + } + }, + "required": [ + "eval_id", + "deleted" + ], + "title": "DeleteEvalResponse", + "type": "object" + }, + "Eval": { + "description": "Represents an evaluation from the OpenAI Evals API", + "properties": { + "created_at": { + "title": "Created At", + "type": "integer" + }, + "data_source_config": { + "additionalProperties": true, + "title": "Data Source Config", + "type": "object" + }, + "id": { + "title": "Id", + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Metadata" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "object": { + "default": "eval", + "title": "Object", + "type": "string" + }, + "testing_criteria": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Testing Criteria", + "type": "array" + }, + "updated_at": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "id", + "created_at", + "data_source_config", + "testing_criteria" + ], + "title": "Eval", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListEvalsResponse": { + "description": "Response from listing evaluations", + "properties": { + "data": { + "items": { + "$ref": "#/components/schemas/Eval" + }, + "title": "Data", + "type": "array" + }, + "first_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "First Id" + }, + "has_more": { + "default": false, + "title": "Has More", + "type": "boolean" + }, + "last_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Id" + }, + "object": { + "default": "list", + "title": "Object", + "type": "string" + } + }, + "required": [ + "data" + ], + "title": "ListEvalsResponse", + "type": "object" + }, + "ListRunsResponse": { + "description": "Response from listing runs", + "properties": { + "data": { + "items": { + "$ref": "#/components/schemas/Run" + }, + "title": "Data", + "type": "array" + }, + "first_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "First Id" + }, + "has_more": { + "default": false, + "title": "Has More", + "type": "boolean" + }, + "last_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Id" + }, + "object": { + "default": "list", + "title": "Object", + "type": "string" + } + }, + "required": [ + "data" + ], + "title": "ListRunsResponse", + "type": "object" + }, + "PerTestingCriteriaResult": { + "description": "Results for a specific testing criteria", + "properties": { + "average_score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Average Score" + }, + "result_counts": { + "$ref": "#/components/schemas/ResultCounts" + }, + "testing_criteria_index": { + "title": "Testing Criteria Index", + "type": "integer" + } + }, + "required": [ + "testing_criteria_index", + "result_counts" + ], + "title": "PerTestingCriteriaResult", + "type": "object" + }, + "ResultCounts": { + "description": "Result counts for a run", + "properties": { + "error": { + "default": 0, + "title": "Error", + "type": "integer" + }, + "failed": { + "default": 0, + "title": "Failed", + "type": "integer" + }, + "passed": { + "default": 0, + "title": "Passed", + "type": "integer" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "total" + ], + "title": "ResultCounts", + "type": "object" + }, + "Run": { + "description": "Represents a run from the OpenAI Evals API", + "properties": { + "completed_at": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Completed At" + }, + "created_at": { + "title": "Created At", + "type": "integer" + }, + "data_source": { + "additionalProperties": true, + "title": "Data Source", + "type": "object" + }, + "error": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "eval_id": { + "title": "Eval Id", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Metadata" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "object": { + "default": "eval.run", + "title": "Object", + "type": "string" + }, + "per_model_usage": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Per Model Usage" + }, + "per_testing_criteria_results": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/PerTestingCriteriaResult" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Per Testing Criteria Results" + }, + "report_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Report Url" + }, + "result_counts": { + "anyOf": [ + { + "additionalProperties": { + "type": "integer" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Result Counts" + }, + "shared_with_openai": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Shared With Openai" + }, + "started_at": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Started At" + }, + "status": { + "enum": [ + "queued", + "running", + "completed", + "failed", + "cancelled" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "id", + "created_at", + "status", + "data_source", + "eval_id" + ], + "title": "Run", + "type": "object" + }, + "RunDeleteResponse": { + "description": "Response from deleting a run", + "properties": { + "deleted": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "title": "Deleted" + }, + "object": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "eval.run.deleted", + "title": "Object" + }, + "run_id": { + "title": "Run Id", + "type": "string" + } + }, + "required": [ + "run_id" + ], + "title": "RunDeleteResponse", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/v1/evals": { + "get": { + "description": "List evaluations with pagination.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals?limit=10\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: ListEvalsResponse with list of evaluations", + "operationId": "list_evals_v1_evals_get", + "parameters": [ + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Order" + } + }, + { + "in": "query", + "name": "order_by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Order By" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListEvalsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Evals", + "tags": [ + "evals" + ] + }, + "post": { + "description": "Create a new evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals\" -H \"Authorization: Bearer your-key\" -H \"Content-Type: application/json\" -d '{\n \"name\": \"Test Eval\",\n \"data_source_config\": {\"type\": \"file\", \"file_id\": \"file-abc123\"},\n \"testing_criteria\": {\"graders\": [{\"type\": \"llm_as_judge\"}]}\n }'\n```\n\nReturns: Eval object with id, status, timestamps, etc.", + "operationId": "create_eval_v1_evals_post", + "parameters": [ + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Eval" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Eval", + "tags": [ + "evals" + ] + } + }, + "/v1/evals/{eval_id}": { + "delete": { + "description": "Delete an evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X DELETE \"http://localhost:4000/v1/evals/eval_123\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: DeleteEvalResponse with deletion confirmation", + "operationId": "delete_eval_v1_evals__eval_id__delete", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteEvalResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Eval", + "tags": [ + "evals" + ] + }, + "get": { + "description": "Get a specific evaluation by ID.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals/eval_123\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: Eval object", + "operationId": "get_eval_v1_evals__eval_id__get", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Eval" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Eval", + "tags": [ + "evals" + ] + }, + "post": { + "description": "Update an evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123\" -H \"Authorization: Bearer your-key\" -H \"Content-Type: application/json\" -d '{\"name\": \"Updated Name\"}'\n```\n\nReturns: Updated Eval object", + "operationId": "update_eval_v1_evals__eval_id__post", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Eval" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Eval", + "tags": [ + "evals" + ] + } + }, + "/v1/evals/{eval_id}/cancel": { + "post": { + "description": "Cancel a running evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123/cancel\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: CancelEvalResponse with cancellation confirmation", + "operationId": "cancel_eval_v1_evals__eval_id__cancel_post", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CancelEvalResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Cancel Eval", + "tags": [ + "evals" + ] + } + }, + "/v1/evals/{eval_id}/runs": { + "get": { + "description": "List all runs for an evaluation with pagination.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals/eval_123/runs?limit=10\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: ListRunsResponse with list of runs", + "operationId": "list_runs_v1_evals__eval_id__runs_get", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Order" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListRunsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Runs", + "tags": [ + "evals" + ] + }, + "post": { + "description": "Create a new run for an evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n- Pass model via completion.model: `{\"completion\": {\"model\": \"gpt-4-account-1\"}}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123/runs\" -H \"Authorization: Bearer your-key\" -H \"Content-Type: application/json\" -d '{\n \"data_source\": {\"type\": \"dataset\", \"dataset_id\": \"dataset_123\"},\n \"completion\": {\"model\": \"gpt-4\", \"temperature\": 0.7}\n }'\n```\n\nReturns: Run object with id, status, timestamps, etc.", + "operationId": "create_run_v1_evals__eval_id__runs_post", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Run" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Run", + "tags": [ + "evals" + ] + } + }, + "/v1/evals/{eval_id}/runs/{run_id}": { + "delete": { + "description": "Delete a run.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl -X DELETE \"http://localhost:4000/v1/evals/eval_123/runs/run_456\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: RunDeleteResponse with deletion confirmation", + "operationId": "delete_run_v1_evals__eval_id__runs__run_id__delete", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RunDeleteResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Run", + "tags": [ + "evals" + ] + }, + "get": { + "description": "Get a specific run by ID.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals/eval_123/runs/run_456\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: Run object with full details", + "operationId": "get_run_v1_evals__eval_id__runs__run_id__get", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Run" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Run", + "tags": [ + "evals" + ] + }, + "post": { + "description": "Cancel a running run.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123/runs/run_456/cancel\" -H \"Authorization: Bearer your-key\"\n```\n\nReturns: CancelRunResponse with cancellation confirmation", + "operationId": "cancel_run_v1_evals__eval_id__runs__run_id__post", + "parameters": [ + { + "in": "path", + "name": "eval_id", + "required": true, + "schema": { + "title": "Eval Id", + "type": "string" + } + }, + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + }, + { + "in": "query", + "name": "custom_llm_provider", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "openai", + "title": "Custom Llm Provider" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CancelRunResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Cancel Run", + "tags": [ + "evals" + ] + } + } + } + }, + "guardrails": { + "components": { + "schemas": { + "ApplyGuardrailRequest": { + "properties": { + "entities": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/PiiEntityType" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Entities" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "input_type": { + "default": "request", + "title": "Input Type", + "type": "string" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "messages": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Messages" + }, + "text": { + "title": "Text", + "type": "string" + } + }, + "required": [ + "guardrail_name", + "text" + ], + "title": "ApplyGuardrailRequest", + "type": "object" + }, + "ApplyGuardrailResponse": { + "properties": { + "response_text": { + "title": "Response Text", + "type": "string" + } + }, + "required": [ + "response_text" + ], + "title": "ApplyGuardrailResponse", + "type": "object" + }, + "BaseLitellmParams-Input": { + "additionalProperties": true, + "properties": { + "additional_provider_specific_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Additional provider-specific parameters for generic guardrail APIs", + "title": "Additional Provider Specific Params" + }, + "api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the guardrail service API", + "title": "Api Base" + }, + "api_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional custom API endpoint for Model Armor", + "title": "Api Endpoint" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "API key for the guardrail service", + "title": "Api Key" + }, + "blocked_words": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/BlockedWord" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of blocked words with individual actions", + "title": "Blocked Words" + }, + "blocked_words_file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to YAML file containing blocked_words list", + "title": "Blocked Words File" + }, + "categories": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterCategoryConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of prebuilt categories to enable (harmful_*, bias_*)", + "title": "Categories" + }, + "category_thresholds": { + "anyOf": [ + { + "$ref": "#/components/schemas/LakeraCategoryThresholds" + }, + { + "type": "null" + } + ], + "description": "Threshold configuration for Lakera guardrail categories" + }, + "credentials": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to Google Cloud credentials JSON file or JSON string", + "title": "Credentials" + }, + "custom_code": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Python-like code containing the apply_guardrail function for custom guardrail logic", + "title": "Custom Code" + }, + "default_on": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Whether the guardrail is enabled by default", + "title": "Default On" + }, + "detect_secrets_config": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Configuration for detect-secrets guardrail", + "title": "Detect Secrets Config" + }, + "end_session_after_n_fails": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: automatically close the session after this many guardrail violations.", + "title": "End Session After N Fails" + }, + "experimental_use_latest_role_message_only": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "When True, guardrails only receive the latest message for the relevant role (e.g., newest user input pre-call, newest assistant output post-call)", + "title": "Experimental Use Latest Role Message Only" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Header names to forward from the client request to the guardrail (e.g. x-request-id). Only these headers' values are sent; others may be omitted or sent as [present]. Used by generic_guardrail_api (similar to MCP extra_headers).", + "title": "Extra Headers" + }, + "fail_on_error": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to fail the request if Model Armor encounters an error", + "title": "Fail On Error" + }, + "guard_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of the guardrail in guardrails.ai", + "title": "Guard Name" + }, + "keyword_redaction_tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Tag to use for keyword redaction", + "title": "Keyword Redaction Tag" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Google Cloud location/region (e.g., us-central1)", + "title": "Location" + }, + "mask_request_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask request content if guardrail makes any changes", + "title": "Mask Request Content" + }, + "mask_response_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask response content if guardrail makes any changes", + "title": "Mask Response Content" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional field if guardrail requires a 'model' parameter", + "title": "Model" + }, + "on_violation": { + "anyOf": [ + { + "enum": [ + "warn", + "end_session" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: 'warn' speaks the violation message and continues; 'end_session' speaks the message and closes the connection.", + "title": "On Violation" + }, + "pangea_input_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for input (LLM request)", + "title": "Pangea Input Recipe" + }, + "pangea_output_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for output (LLM response)", + "title": "Pangea Output Recipe" + }, + "pattern_redaction_format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Format string for pattern redaction (use {pattern_name} placeholder)", + "title": "Pattern Redaction Format" + }, + "patterns": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterPattern" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of patterns (prebuilt or custom regex) to detect", + "title": "Patterns" + }, + "realtime_violation_message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The message the bot speaks aloud when a /v1/realtime guardrail fires. Falls back to violation_message_template if not set.", + "title": "Realtime Violation Message" + }, + "severity_threshold": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Minimum severity to block (high, medium, low)", + "title": "Severity Threshold" + }, + "skip_system_message_in_guardrail": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "When True, unified guardrails skip system-role messages when building evaluation inputs (texts and structured_messages). When False, system messages are included even if litellm_settings sets a global skip. When None, use the global litellm.skip_system_message_in_guardrail setting.", + "title": "Skip System Message In Guardrail" + }, + "template_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The ID of your Model Armor template", + "title": "Template Id" + }, + "unreachable_fallback": { + "default": "fail_closed", + "description": "Behavior when a guardrail endpoint is unreachable due to network errors. NOTE: This is currently only implemented by guardrail='generic_guardrail_api'. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed.", + "enum": [ + "fail_closed", + "fail_open" + ], + "title": "Unreachable Fallback", + "type": "string" + }, + "violation_message_template": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Custom message when a guardrail blocks an action. Supports placeholders like {tool_name}, {rule_id}, and {default_message}.", + "title": "Violation Message Template" + } + }, + "title": "BaseLitellmParams", + "type": "object" + }, + "BaseLitellmParams-Output": { + "additionalProperties": true, + "properties": { + "additional_provider_specific_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Additional provider-specific parameters for generic guardrail APIs", + "title": "Additional Provider Specific Params" + }, + "api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the guardrail service API", + "title": "Api Base" + }, + "api_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional custom API endpoint for Model Armor", + "title": "Api Endpoint" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "API key for the guardrail service", + "title": "Api Key" + }, + "blocked_words": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/BlockedWord" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of blocked words with individual actions", + "title": "Blocked Words" + }, + "blocked_words_file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to YAML file containing blocked_words list", + "title": "Blocked Words File" + }, + "categories": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterCategoryConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of prebuilt categories to enable (harmful_*, bias_*)", + "title": "Categories" + }, + "category_thresholds": { + "anyOf": [ + { + "$ref": "#/components/schemas/LakeraCategoryThresholds" + }, + { + "type": "null" + } + ], + "description": "Threshold configuration for Lakera guardrail categories" + }, + "credentials": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to Google Cloud credentials JSON file or JSON string", + "title": "Credentials" + }, + "custom_code": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Python-like code containing the apply_guardrail function for custom guardrail logic", + "title": "Custom Code" + }, + "default_on": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Whether the guardrail is enabled by default", + "title": "Default On" + }, + "detect_secrets_config": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Configuration for detect-secrets guardrail", + "title": "Detect Secrets Config" + }, + "end_session_after_n_fails": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: automatically close the session after this many guardrail violations.", + "title": "End Session After N Fails" + }, + "experimental_use_latest_role_message_only": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "When True, guardrails only receive the latest message for the relevant role (e.g., newest user input pre-call, newest assistant output post-call)", + "title": "Experimental Use Latest Role Message Only" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Header names to forward from the client request to the guardrail (e.g. x-request-id). Only these headers' values are sent; others may be omitted or sent as [present]. Used by generic_guardrail_api (similar to MCP extra_headers).", + "title": "Extra Headers" + }, + "fail_on_error": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to fail the request if Model Armor encounters an error", + "title": "Fail On Error" + }, + "guard_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of the guardrail in guardrails.ai", + "title": "Guard Name" + }, + "keyword_redaction_tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Tag to use for keyword redaction", + "title": "Keyword Redaction Tag" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Google Cloud location/region (e.g., us-central1)", + "title": "Location" + }, + "mask_request_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask request content if guardrail makes any changes", + "title": "Mask Request Content" + }, + "mask_response_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask response content if guardrail makes any changes", + "title": "Mask Response Content" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional field if guardrail requires a 'model' parameter", + "title": "Model" + }, + "on_violation": { + "anyOf": [ + { + "enum": [ + "warn", + "end_session" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: 'warn' speaks the violation message and continues; 'end_session' speaks the message and closes the connection.", + "title": "On Violation" + }, + "pangea_input_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for input (LLM request)", + "title": "Pangea Input Recipe" + }, + "pangea_output_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for output (LLM response)", + "title": "Pangea Output Recipe" + }, + "pattern_redaction_format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Format string for pattern redaction (use {pattern_name} placeholder)", + "title": "Pattern Redaction Format" + }, + "patterns": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterPattern" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of patterns (prebuilt or custom regex) to detect", + "title": "Patterns" + }, + "realtime_violation_message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The message the bot speaks aloud when a /v1/realtime guardrail fires. Falls back to violation_message_template if not set.", + "title": "Realtime Violation Message" + }, + "severity_threshold": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Minimum severity to block (high, medium, low)", + "title": "Severity Threshold" + }, + "skip_system_message_in_guardrail": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "When True, unified guardrails skip system-role messages when building evaluation inputs (texts and structured_messages). When False, system messages are included even if litellm_settings sets a global skip. When None, use the global litellm.skip_system_message_in_guardrail setting.", + "title": "Skip System Message In Guardrail" + }, + "template_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The ID of your Model Armor template", + "title": "Template Id" + }, + "unreachable_fallback": { + "default": "fail_closed", + "description": "Behavior when a guardrail endpoint is unreachable due to network errors. NOTE: This is currently only implemented by guardrail='generic_guardrail_api'. 'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed.", + "enum": [ + "fail_closed", + "fail_open" + ], + "title": "Unreachable Fallback", + "type": "string" + }, + "violation_message_template": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Custom message when a guardrail blocks an action. Supports placeholders like {tool_name}, {rule_id}, and {default_message}.", + "title": "Violation Message Template" + } + }, + "title": "BaseLitellmParams", + "type": "object" + }, + "BlockedWord": { + "description": "Represents a blocked word with its action and optional description", + "properties": { + "action": { + "$ref": "#/components/schemas/ContentFilterAction", + "description": "Action to take when keyword is detected (BLOCK or MASK)" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional description explaining why this keyword is sensitive", + "title": "Description" + }, + "keyword": { + "description": "The keyword to block or mask", + "title": "Keyword", + "type": "string" + } + }, + "required": [ + "keyword", + "action" + ], + "title": "BlockedWord", + "type": "object" + }, + "ContentFilterAction": { + "description": "Action to take when content filter detects a match", + "enum": [ + "BLOCK", + "MASK" + ], + "title": "ContentFilterAction", + "type": "string" + }, + "ContentFilterCategoryConfig": { + "additionalProperties": true, + "description": "category: \"harmful_self_harm\"\n enabled: true\n action: \"BLOCK\"\n severity_threshold: \"medium\"\n category_file: \"/path/to/custom_file.yaml\" # optional override", + "properties": { + "action": { + "description": "The action to take when the category is detected", + "enum": [ + "BLOCK", + "MASK" + ], + "title": "Action", + "type": "string" + }, + "category": { + "description": "The category to detect", + "title": "Category", + "type": "string" + }, + "category_file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional override. Use your own category file instead of the default one.", + "title": "Category File" + }, + "enabled": { + "default": true, + "description": "Whether the category is enabled", + "title": "Enabled", + "type": "boolean" + }, + "severity_threshold": { + "default": "medium", + "description": "The severity threshold to detect the category", + "enum": [ + "high", + "medium", + "low" + ], + "title": "Severity Threshold", + "type": "string" + } + }, + "required": [ + "category", + "action" + ], + "title": "ContentFilterCategoryConfig", + "type": "object" + }, + "ContentFilterPattern": { + "description": "Represents a content filter pattern (prebuilt or custom regex)", + "properties": { + "action": { + "$ref": "#/components/schemas/ContentFilterAction", + "description": "Action to take when pattern matches (BLOCK or MASK)" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name for this pattern (used in logging and error messages)", + "title": "Name" + }, + "pattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Custom regex pattern. Required if pattern_type is 'regex'", + "title": "Pattern" + }, + "pattern_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of prebuilt pattern (e.g., 'us_ssn', 'credit_card'). Required if pattern_type is 'prebuilt'", + "title": "Pattern Name" + }, + "pattern_type": { + "description": "Type of pattern: 'prebuilt' for predefined patterns or 'regex' for custom", + "enum": [ + "prebuilt", + "regex" + ], + "title": "Pattern Type", + "type": "string" + } + }, + "required": [ + "pattern_type", + "action" + ], + "title": "ContentFilterPattern", + "type": "object" + }, + "CreateGuardrailRequest": { + "properties": { + "guardrail": { + "$ref": "#/components/schemas/Guardrail" + } + }, + "required": [ + "guardrail" + ], + "title": "CreateGuardrailRequest", + "type": "object" + }, + "GUARDRAIL_DEFINITION_LOCATION": { + "enum": [ + "db", + "config" + ], + "title": "GUARDRAIL_DEFINITION_LOCATION", + "type": "string" + }, + "GraySwanGuardrailConfigModelOptionalParams": { + "description": "Optional parameters for the Gray Swan guardrail.", + "properties": { + "categories": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Default Gray Swan category definitions to send with each request.", + "title": "Categories" + }, + "fail_open": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "If true (default), errors contacting Gray Swan are logged and the request proceeds. If false, errors propagate and block the request.", + "title": "Fail Open" + }, + "guardrail_timeout": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": 30.0, + "description": "Timeout in seconds for calling the Gray Swan guardrail service.", + "title": "Guardrail Timeout" + }, + "on_flagged_action": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "passthrough", + "description": "Action when a violation is detected: 'block' rejects the call (400 error), 'monitor' logs only, 'passthrough' replaces response content with violation message (200 status).", + "title": "On Flagged Action" + }, + "policy_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Gray Swan policy identifier to apply during monitoring.", + "title": "Policy Id" + }, + "reasoning_mode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Gray Swan reasoning mode override. Accepted values: 'off', 'hybrid', 'thinking'.", + "title": "Reasoning Mode" + }, + "violation_threshold": { + "anyOf": [ + { + "maximum": 1.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": 0.5, + "description": "Threshold between 0 and 1 at which Gray Swan violations trigger the configured action.", + "title": "Violation Threshold" + } + }, + "title": "GraySwanGuardrailConfigModelOptionalParams", + "type": "object" + }, + "Guardrail": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "guardrail_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Guardrail Id" + }, + "guardrail_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Guardrail Info" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "litellm_params": { + "$ref": "#/components/schemas/LitellmParams" + }, + "policy_template": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Policy Template" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "guardrail_name", + "litellm_params" + ], + "title": "Guardrail", + "type": "object" + }, + "GuardrailInfoResponse": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "guardrail_definition_location": { + "$ref": "#/components/schemas/GUARDRAIL_DEFINITION_LOCATION", + "default": "config" + }, + "guardrail_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Guardrail Id" + }, + "guardrail_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Guardrail Info" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "litellm_params": { + "anyOf": [ + { + "$ref": "#/components/schemas/BaseLitellmParams-Output" + }, + { + "type": "null" + } + ] + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "guardrail_name" + ], + "title": "GuardrailInfoResponse", + "type": "object" + }, + "GuardrailSubmissionItem": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "guardrail_id": { + "title": "Guardrail Id", + "type": "string" + }, + "guardrail_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Guardrail Info" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "litellm_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Litellm Params" + }, + "reviewed_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reviewed At" + }, + "status": { + "title": "Status", + "type": "string" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted At" + }, + "submitted_by_email": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted By Email" + }, + "submitted_by_user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted By User Id" + }, + "team_guardrail": { + "default": false, + "title": "Team Guardrail", + "type": "boolean" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "guardrail_id", + "guardrail_name", + "status" + ], + "title": "GuardrailSubmissionItem", + "type": "object" + }, + "GuardrailSubmissionSummary": { + "properties": { + "active": { + "title": "Active", + "type": "integer" + }, + "pending_review": { + "title": "Pending Review", + "type": "integer" + }, + "rejected": { + "title": "Rejected", + "type": "integer" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "total", + "pending_review", + "active", + "rejected" + ], + "title": "GuardrailSubmissionSummary", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "LakeraCategoryThresholds": { + "additionalProperties": true, + "properties": { + "jailbreak": { + "title": "Jailbreak", + "type": "number" + }, + "prompt_injection": { + "title": "Prompt Injection", + "type": "number" + } + }, + "title": "LakeraCategoryThresholds", + "type": "object" + }, + "ListGuardrailSubmissionsResponse": { + "properties": { + "submissions": { + "items": { + "$ref": "#/components/schemas/GuardrailSubmissionItem" + }, + "title": "Submissions", + "type": "array" + }, + "summary": { + "$ref": "#/components/schemas/GuardrailSubmissionSummary" + } + }, + "required": [ + "submissions", + "summary" + ], + "title": "ListGuardrailSubmissionsResponse", + "type": "object" + }, + "ListGuardrailsResponse": { + "properties": { + "guardrails": { + "items": { + "$ref": "#/components/schemas/GuardrailInfoResponse" + }, + "title": "Guardrails", + "type": "array" + } + }, + "required": [ + "guardrails" + ], + "title": "ListGuardrailsResponse", + "type": "object" + }, + "LitellmParams": { + "additionalProperties": true, + "properties": { + "action": { + "default": "block", + "description": "'block' raises an error; 'mask' replaces the code block with a placeholder.", + "enum": [ + "block", + "mask" + ], + "title": "Action", + "type": "string" + }, + "additional_provider_specific_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Additional provider-specific parameters for generic guardrail APIs", + "title": "Additional Provider Specific Params" + }, + "akto_account_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Akto account ID for multi-tenant deployments. Env: AKTO_ACCOUNT_ID. Default: '1000000'.", + "title": "Akto Account Id" + }, + "akto_api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "API key for Akto. Env: AKTO_API_KEY.", + "title": "Akto Api Key" + }, + "akto_base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Akto Guardrail API Base URL. Env: AKTO_GUARDRAIL_API_BASE.", + "examples": [ + "http://localhost:9090", + "https://akto-ingestion.example.com" + ], + "title": "Akto Base Url" + }, + "akto_vxlan_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Akto VXLAN ID. Env: AKTO_VXLAN_ID. Default: '0'.", + "title": "Akto Vxlan Id" + }, + "anonymize_input": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "If True, replaces sensitive content with anonymized version when only PII/PCI/secrets are detected. Only applies in blocking mode. Defaults to False if not provided", + "title": "Anonymize Input" + }, + "api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the Lakera AI API", + "title": "Api Base" + }, + "api_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional custom API endpoint for Model Armor", + "title": "Api Endpoint" + }, + "api_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The Hiddenlayer API Id for the Hiddenlayer API. If not provided, the `HIDDENLAYER_CLIENT_ID` environment variable is checked or https://api.hiddenlayer.ai is used.", + "title": "Api Id" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "API key for the Lakera AI service", + "title": "Api Key" + }, + "api_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "v1", + "description": "API version for Javelin service", + "title": "Api Version" + }, + "application": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Application name for Javelin service", + "title": "Application" + }, + "application_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Application ID for Noma Security. Defaults to 'litellm' if not provided", + "title": "Application Id" + }, + "assertions": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Custom assertions to validate against the output. Each assertion is a string describing a condition.", + "title": "Assertions" + }, + "async_mode": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Set to True to request asynchronous analysis (sets `plr_async` header). Defaults to provider behaviour when omitted.", + "title": "Async Mode" + }, + "auth_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Authorization bearer token for IBM Guardrails API. Reads from IBM_GUARDRAILS_AUTH_TOKEN env var if None.", + "title": "Auth Token" + }, + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS access key ID for authentication", + "title": "Aws Access Key Id" + }, + "aws_bedrock_runtime_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS Bedrock runtime endpoint URL", + "title": "Aws Bedrock Runtime Endpoint" + }, + "aws_profile_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS profile name for credential retrieval", + "title": "Aws Profile Name" + }, + "aws_region_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS region where your guardrail is deployed", + "title": "Aws Region Name" + }, + "aws_role_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS role name for assuming roles", + "title": "Aws Role Name" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS secret access key for authentication", + "title": "Aws Secret Access Key" + }, + "aws_session_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of the AWS session", + "title": "Aws Session Name" + }, + "aws_session_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS session token for temporary credentials", + "title": "Aws Session Token" + }, + "aws_sts_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "AWS STS endpoint URL", + "title": "Aws Sts Endpoint" + }, + "aws_web_identity_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Web identity token for AWS role assumption", + "title": "Aws Web Identity Token" + }, + "base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the IBM Guardrails server", + "title": "Base Url" + }, + "block_failures": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "If True, blocks requests on API failures. Defaults to True if not provided", + "title": "Block Failures" + }, + "block_on_error": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Whether to block the request when the PromptGuard API is unreachable. Defaults to true (fail-closed). Set to false for fail-open behaviour.", + "title": "Block On Error" + }, + "block_on_violation": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to block requests when violations are detected. Defaults to True.", + "title": "Block On Violation" + }, + "blocked_languages": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Language tags to block (e.g. python, javascript, bash). Empty or None = block all fenced code blocks.", + "options": [ + "python", + "javascript", + "typescript", + "bash", + "ruby", + "go", + "java", + "csharp", + "php", + "c", + "cpp", + "rust", + "sql" + ], + "title": "Blocked Languages", + "ui_type": "multiselect" + }, + "blocked_words": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/BlockedWord" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of blocked words with individual actions", + "title": "Blocked Words" + }, + "blocked_words_file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to YAML file containing blocked_words list", + "title": "Blocked Words File" + }, + "breakdown": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to include breakdown in the response", + "title": "Breakdown" + }, + "categories": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterCategoryConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of prebuilt categories to enable (harmful_*, bias_*)", + "title": "Categories" + }, + "category_thresholds": { + "anyOf": [ + { + "$ref": "#/components/schemas/LakeraCategoryThresholds" + }, + { + "type": "null" + } + ], + "description": "Threshold configuration for Lakera guardrail categories" + }, + "confidence_threshold": { + "default": 0.5, + "default_value": 0.5, + "description": "Only block or mask when detection confidence >= this value; below threshold, allow or log_only.", + "max": 1.0, + "maximum": 1.0, + "min": 0.0, + "minimum": 0.0, + "step": 0.1, + "title": "Confidence Threshold", + "type": "number", + "ui_type": "percentage" + }, + "config": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Additional configuration for the guardrail", + "title": "Config" + }, + "content_moderation_check": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable content moderation to check for harmful content (harassment, hate speech, etc.).", + "title": "Content Moderation Check" + }, + "credentials": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to Google Cloud credentials JSON file or JSON string", + "title": "Credentials" + }, + "custom_code": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Python-like code containing the apply_guardrail function for custom guardrail logic", + "title": "Custom Code" + }, + "default_action": { + "default": "deny", + "description": "Fallback decision when no rule matches", + "enum": [ + "allow", + "deny" + ], + "title": "Default Action", + "type": "string" + }, + "default_on": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Whether the guardrail is enabled by default", + "title": "Default On" + }, + "deployment_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The EnkryptAI deployment name to use. Sent via X-Enkrypt-Deployment header.", + "title": "Deployment Name" + }, + "detect_execution_intent": { + "default": true, + "description": "When True, block only when user intent is to run/execute; allow when intent is explain/refactor/don't run. Also block text-only execution requests (e.g. 'run `ls`', 'read /etc/passwd').", + "title": "Detect Execution Intent", + "type": "boolean" + }, + "detect_secrets_config": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Configuration for detect-secrets guardrail", + "title": "Detect Secrets Config" + }, + "detector_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of the detector inside the server (e.g., 'jailbreak-detector')", + "title": "Detector Id" + }, + "detectors": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Dictionary of detector configurations (e.g., {'nsfw': {'enabled': True}, 'toxicity': {'enabled': True}}).", + "title": "Detectors" + }, + "dev_info": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to include developer information in the response", + "title": "Dev Info" + }, + "disable_exception_on_block": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If True, will not raise an exception when the guardrail is blocked. Useful for OpenWebUI where exceptions can end the chat flow.", + "title": "Disable Exception On Block" + }, + "end_session_after_n_fails": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: automatically close the session after this many guardrail violations.", + "title": "End Session After N Fails" + }, + "evaluation_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Pre-configured evaluation ID from Qualifire dashboard. When provided, uses invoke_evaluation() instead of evaluate().", + "title": "Evaluation Id" + }, + "experimental_use_latest_role_message_only": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "When True, guardrails only receive the latest message for the relevant role (e.g., newest user input pre-call, newest assistant output post-call)", + "title": "Experimental Use Latest Role Message Only" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Header names to forward from the client request to the guardrail (e.g. x-request-id). Only these headers' values are sent; others may be omitted or sent as [present]. Used by generic_guardrail_api (similar to MCP extra_headers).", + "title": "Extra Headers" + }, + "fail_on_error": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to fail the request if Model Armor encounters an error", + "title": "Fail On Error" + }, + "grounding_check": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable grounding verification to ensure output is grounded in provided context.", + "title": "Grounding Check" + }, + "grounding_strictness": { + "anyOf": [ + { + "enum": [ + "BALANCED", + "STRICT" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Strictness level for XecGuard context-grounding validation. 'BALANCED' (default) treats INCOMPLETE answers as SAFE; 'STRICT' flags them as UNSAFE. Grounding only runs in post_call when `metadata.xecguard_grounding_documents` is provided.", + "title": "Grounding Strictness" + }, + "guard_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of the Javelin guard to use", + "title": "Guard Name" + }, + "guardrail": { + "description": "The type of guardrail integration to use", + "title": "Guardrail", + "type": "string" + }, + "guardrailIdentifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The ID of your guardrail on Bedrock", + "title": "Guardrailidentifier" + }, + "guardrailVersion": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The version of your Bedrock guardrail (e.g., DRAFT or version number)", + "title": "Guardrailversion" + }, + "guardrail_timeout": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "HTTP timeout in seconds. Default: 5.", + "title": "Guardrail Timeout" + }, + "hallucinations_check": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable hallucination detection to detect factual inaccuracies.", + "title": "Hallucinations Check" + }, + "include_evidence": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Include detailed evidence payloads in responses (sets `plr_evidence` header).", + "title": "Include Evidence" + }, + "include_scanners": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Include scanner category summaries in responses (sets `plr_scanners` header).", + "title": "Include Scanners" + }, + "is_detector_server": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Boolean flag to determine if calling a detector server (True) or the FMS Orchestrator (False). Defaults to True.", + "title": "Is Detector Server" + }, + "keyword_redaction_tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Tag to use for keyword redaction", + "title": "Keyword Redaction Tag" + }, + "lasso_conversation_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Conversation ID for the Lasso guardrail", + "title": "Lasso Conversation Id" + }, + "lasso_user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "User ID for the Lasso guardrail", + "title": "Lasso User Id" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Google Cloud location/region (e.g., us-central1)", + "title": "Location" + }, + "mask": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Enable content masking using Lasso classifix API", + "title": "Mask" + }, + "mask_request_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask request content if guardrail makes any changes", + "title": "Mask Request Content" + }, + "mask_response_content": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Will mask response content if guardrail makes any changes", + "title": "Mask Response Content" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Additional metadata to include in the request", + "title": "Metadata" + }, + "mock_redacted_text": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Mock redacted text for testing", + "title": "Mock Redacted Text" + }, + "mode": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "$ref": "#/components/schemas/Mode" + } + ], + "description": "When to apply the guardrail (pre_call, post_call, during_call, logging_only)", + "title": "Mode" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Optional field if guardrail requires a 'model' parameter", + "title": "Model" + }, + "monitor_mode": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "If True, logs violations without blocking. Defaults to False if not provided", + "title": "Monitor Mode" + }, + "on_disallowed_action": { + "default": "block", + "description": "Choose whether disallowed tools block the request or get rewritten out of the payload", + "enum": [ + "block", + "rewrite" + ], + "title": "On Disallowed Action", + "type": "string" + }, + "on_flagged": { + "anyOf": [ + { + "enum": [ + "block", + "monitor" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": "block", + "description": "Action to take when content is flagged: 'block' (raise exception) or 'monitor' (log only)", + "title": "On Flagged" + }, + "on_flagged_action": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "monitor", + "description": "Action to take when content is flagged: 'block' (raise exception) or 'monitor' (log only)", + "title": "On Flagged Action" + }, + "on_violation": { + "anyOf": [ + { + "enum": [ + "warn", + "end_session" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "For /v1/realtime sessions: 'warn' speaks the violation message and continues; 'end_session' speaks the message and closes the connection.", + "title": "On Violation" + }, + "optional_params": { + "anyOf": [ + { + "$ref": "#/components/schemas/GraySwanGuardrailConfigModelOptionalParams" + }, + { + "type": "null" + } + ], + "description": "Optional parameters for the guardrail" + }, + "output_parse_pii": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "When True, LiteLLM will replace the masked text with the original text in the response", + "title": "Output Parse Pii", + "ui_type": "bool" + }, + "pangea_input_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for input (LLM request)", + "title": "Pangea Input Recipe" + }, + "pangea_output_recipe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Recipe for output (LLM response)", + "title": "Pangea Output Recipe" + }, + "pattern_redaction_format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Format string for pattern redaction (use {pattern_name} placeholder)", + "title": "Pattern Redaction Format" + }, + "patterns": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ContentFilterPattern" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of patterns (prebuilt or custom regex) to detect", + "title": "Patterns" + }, + "payload": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to include payload in the response", + "title": "Payload" + }, + "persist_session": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Controls Pillar session persistence (sets `plr_persist` header). Set to False to disable persistence.", + "title": "Persist Session" + }, + "pii_check": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable PII (Personally Identifiable Information) detection.", + "title": "Pii Check" + }, + "pii_entities_config": { + "anyOf": [ + { + "additionalProperties": { + "$ref": "#/components/schemas/PiiAction" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Configuration for PII entity types and actions", + "title": "Pii Entities Config" + }, + "policy_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Policy ID for Zscaler AI Guard. Can also be set via ZSCALER_AI_GUARD_POLICY_ID environment variable", + "title": "Policy Id" + }, + "policy_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The EnkryptAI policy name to use. Sent via x-enkrypt-policy header.", + "title": "Policy Name" + }, + "policy_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "XecGuard policies to apply on each scan. Select one or more of the built-in default policies; if none are selected, the guardrail defaults to System Prompt Enforcement + Harmful Content Protection.", + "options": [ + "Default_Policy_SystemPromptEnforcement", + "Default_Policy_GeneralPromptAttackProtection", + "Default_Policy_ContentBiasProtection", + "Default_Policy_HarmfulContentProtection", + "Default_Policy_SkillsProtection", + "Default_Policy_PIISensitiveDataProtection" + ], + "title": "Policy Names", + "ui_type": "multiselect" + }, + "presidio_ad_hoc_recognizers": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Path to a JSON file containing ad-hoc recognizers for Presidio", + "title": "Presidio Ad Hoc Recognizers" + }, + "presidio_analyzer_api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the Presidio analyzer API", + "title": "Presidio Analyzer Api Base" + }, + "presidio_anonymizer_api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Base URL for the Presidio anonymizer API", + "title": "Presidio Anonymizer Api Base" + }, + "presidio_entities_deny_list": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/PiiEntityType" + }, + { + "type": "string" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of entity types to exclude from Presidio detection results. Detections of these types will be silently dropped. Useful for suppressing false positives (e.g., US_DRIVER_LICENSE on coding routes).", + "title": "Presidio Entities Deny List" + }, + "presidio_filter_scope": { + "anyOf": [ + { + "enum": [ + "input", + "output", + "both" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Where to apply Presidio checks: 'input' (user -> model), 'output' (model -> user), or 'both' (default).", + "title": "Presidio Filter Scope" + }, + "presidio_language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "en", + "description": "Language code for Presidio PII analysis (e.g., 'en', 'de', 'es', 'fr')", + "title": "Presidio Language" + }, + "presidio_run_on": { + "anyOf": [ + { + "enum": [ + "input", + "output", + "both" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Where to apply Presidio checks: input, output, or both (default).", + "title": "Presidio Run On" + }, + "presidio_score_thresholds": { + "anyOf": [ + { + "additionalProperties": { + "type": "number" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional per-entity minimum confidence scores for Presidio detections. Entities below the threshold are ignored.", + "title": "Presidio Score Thresholds" + }, + "project_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Project ID for the Lakera AI project", + "title": "Project Id" + }, + "prompt_injections": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable prompt injection detection. Default check if no evaluation_id and no other checks are specified.", + "title": "Prompt Injections" + }, + "realtime_violation_message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The message the bot speaks aloud when a /v1/realtime guardrail fires. Falls back to violation_message_template if not set.", + "title": "Realtime Violation Message" + }, + "rules": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ToolPermissionRule" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Ordered allow/deny rules. Patterns use regex for tool names/types and optional regex constraints on tool arguments.", + "title": "Rules" + }, + "send_user_api_key_alias": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether to send user_API_key_alias in headers", + "title": "Send User Api Key Alias" + }, + "send_user_api_key_team_id": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether to send user_API_key_team_id in headers", + "title": "Send User Api Key Team Id" + }, + "send_user_api_key_user_id": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether to send user_API_key_user_id in headers", + "title": "Send User Api Key User Id" + }, + "severity_threshold": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Minimum severity to block (high, medium, low)", + "title": "Severity Threshold" + }, + "skip_system_message_in_guardrail": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "When True, unified guardrails skip system-role messages when building evaluation inputs (texts and structured_messages). When False, system messages are included even if litellm_settings sets a global skip. When None, use the global litellm.skip_system_message_in_guardrail setting.", + "title": "Skip System Message In Guardrail" + }, + "template_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The ID of your Model Armor template", + "title": "Template Id" + }, + "tool_selection_quality_check": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Enable tool selection quality check to evaluate quality of tool/function calls.", + "title": "Tool Selection Quality Check" + }, + "unreachable_fallback": { + "default": "fail_closed", + "description": "What to do when Akto is unreachable. 'fail_open' = allow, 'fail_closed' = block.", + "enum": [ + "fail_closed", + "fail_open" + ], + "title": "Unreachable Fallback", + "type": "string" + }, + "use_v2": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If True and guardrail='noma', route to the new Noma v2 implementation instead of the legacy implementation.", + "title": "Use V2" + }, + "verify_ssl": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to verify SSL certificates. Defaults to True.", + "title": "Verify Ssl" + }, + "version": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 2, + "description": "Hiddenlayer guardrail version to use.", + "title": "Version" + }, + "violation_message_template": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Custom message when a guardrail blocks an action. Supports placeholders like {tool_name}, {rule_id}, and {default_message}.", + "title": "Violation Message Template" + }, + "xecguard_model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "XecGuard scanning model identifier. Defaults to 'xecguard_v2'.", + "title": "Xecguard Model" + } + }, + "required": [ + "guardrail", + "mode" + ], + "title": "LitellmParams", + "type": "object" + }, + "Mode": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Default mode when no tags match", + "title": "Default" + }, + "tags": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "description": "Tags for the guardrail mode", + "title": "Tags", + "type": "object" + } + }, + "required": [ + "tags" + ], + "title": "Mode", + "type": "object" + }, + "PatchGuardrailRequest": { + "properties": { + "guardrail_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Guardrail Info" + }, + "guardrail_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Guardrail Name" + }, + "litellm_params": { + "anyOf": [ + { + "$ref": "#/components/schemas/BaseLitellmParams-Input" + }, + { + "type": "null" + } + ] + } + }, + "title": "PatchGuardrailRequest", + "type": "object" + }, + "PiiAction": { + "enum": [ + "BLOCK", + "MASK" + ], + "title": "PiiAction", + "type": "string" + }, + "PiiEntityType": { + "enum": [ + "CREDIT_CARD", + "CRYPTO", + "DATE_TIME", + "EMAIL_ADDRESS", + "IBAN_CODE", + "IP_ADDRESS", + "NRP", + "LOCATION", + "PERSON", + "PHONE_NUMBER", + "MEDICAL_LICENSE", + "URL", + "US_BANK_NUMBER", + "US_DRIVER_LICENSE", + "US_ITIN", + "US_PASSPORT", + "US_SSN", + "UK_NHS", + "UK_NINO", + "ES_NIF", + "ES_NIE", + "IT_FISCAL_CODE", + "IT_DRIVER_LICENSE", + "IT_VAT_CODE", + "IT_PASSPORT", + "IT_IDENTITY_CARD", + "PL_PESEL", + "SG_NRIC_FIN", + "SG_UEN", + "AU_ABN", + "AU_ACN", + "AU_TFN", + "AU_MEDICARE", + "IN_PAN", + "IN_AADHAAR", + "IN_VEHICLE_REGISTRATION", + "IN_VOTER", + "IN_PASSPORT", + "FI_PERSONAL_IDENTITY_CODE" + ], + "title": "PiiEntityType", + "type": "string" + }, + "RegisterGuardrailRequest": { + "description": "Request body for POST /guardrails/register. Follows Generic Guardrail API config.", + "properties": { + "guardrail_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Guardrail Info" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "litellm_params": { + "additionalProperties": true, + "title": "Litellm Params", + "type": "object" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + } + }, + "required": [ + "guardrail_name", + "litellm_params" + ], + "title": "RegisterGuardrailRequest", + "type": "object" + }, + "RegisterGuardrailResponse": { + "properties": { + "guardrail_id": { + "title": "Guardrail Id", + "type": "string" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "status": { + "title": "Status", + "type": "string" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted At" + } + }, + "required": [ + "guardrail_id", + "guardrail_name", + "status" + ], + "title": "RegisterGuardrailResponse", + "type": "object" + }, + "TestCustomCodeGuardrailRequest": { + "description": "Request model for testing custom code guardrails.", + "properties": { + "custom_code": { + "title": "Custom Code", + "type": "string" + }, + "input_type": { + "default": "request", + "title": "Input Type", + "type": "string" + }, + "request_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Request Data" + }, + "test_input": { + "additionalProperties": true, + "title": "Test Input", + "type": "object" + } + }, + "required": [ + "custom_code", + "test_input" + ], + "title": "TestCustomCodeGuardrailRequest", + "type": "object" + }, + "TestCustomCodeGuardrailResponse": { + "description": "Response model for testing custom code guardrails.", + "properties": { + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + }, + "error_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error Type" + }, + "result": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Result" + }, + "success": { + "title": "Success", + "type": "boolean" + } + }, + "required": [ + "success" + ], + "title": "TestCustomCodeGuardrailResponse", + "type": "object" + }, + "ToolPermissionRule": { + "description": "A rule defining permission for a specific tool or tool pattern", + "properties": { + "allowed_param_patterns": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional regex map enforcing nested parameter values using dot/[] paths", + "title": "Allowed Param Patterns" + }, + "decision": { + "description": "Whether to allow or deny this tool usage", + "enum": [ + "allow", + "deny" + ], + "title": "Decision", + "type": "string" + }, + "id": { + "description": "Unique identifier for the rule", + "title": "Id", + "type": "string" + }, + "tool_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Regex pattern applied to the tool's function name", + "title": "Tool Name" + }, + "tool_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Regex pattern applied to the tool type (e.g., function)", + "title": "Tool Type" + } + }, + "required": [ + "id", + "decision" + ], + "title": "ToolPermissionRule", + "type": "object" + }, + "UpdateGuardrailRequest": { + "properties": { + "guardrail": { + "$ref": "#/components/schemas/Guardrail" + } + }, + "required": [ + "guardrail" + ], + "title": "UpdateGuardrailRequest", + "type": "object" + }, + "UsageDetailResponse": { + "properties": { + "avgLatency": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avglatency" + }, + "avgScore": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avgscore" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "failRate": { + "title": "Failrate", + "type": "number" + }, + "guardrail_id": { + "title": "Guardrail Id", + "type": "string" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "requestsEvaluated": { + "title": "Requestsevaluated", + "type": "integer" + }, + "status": { + "title": "Status", + "type": "string" + }, + "time_series": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Time Series", + "type": "array" + }, + "trend": { + "title": "Trend", + "type": "string" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "guardrail_id", + "guardrail_name", + "type", + "provider", + "requestsEvaluated", + "failRate", + "avgScore", + "avgLatency", + "status", + "trend", + "description", + "time_series" + ], + "title": "UsageDetailResponse", + "type": "object" + }, + "UsageLogEntry": { + "properties": { + "action": { + "title": "Action", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "input_snippet": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Input Snippet" + }, + "latency_ms": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Latency Ms" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "output_snippet": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Output Snippet" + }, + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reason" + }, + "score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Score" + }, + "timestamp": { + "title": "Timestamp", + "type": "string" + } + }, + "required": [ + "id", + "timestamp", + "action", + "score", + "latency_ms", + "model", + "input_snippet", + "output_snippet", + "reason" + ], + "title": "UsageLogEntry", + "type": "object" + }, + "UsageLogsResponse": { + "properties": { + "logs": { + "items": { + "$ref": "#/components/schemas/UsageLogEntry" + }, + "title": "Logs", + "type": "array" + }, + "page": { + "title": "Page", + "type": "integer" + }, + "page_size": { + "title": "Page Size", + "type": "integer" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "logs", + "total", + "page", + "page_size" + ], + "title": "UsageLogsResponse", + "type": "object" + }, + "UsageOverviewResponse": { + "properties": { + "chart": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Chart", + "type": "array" + }, + "passRate": { + "title": "Passrate", + "type": "number" + }, + "rows": { + "items": { + "$ref": "#/components/schemas/UsageOverviewRow" + }, + "title": "Rows", + "type": "array" + }, + "totalBlocked": { + "title": "Totalblocked", + "type": "integer" + }, + "totalRequests": { + "title": "Totalrequests", + "type": "integer" + } + }, + "required": [ + "rows", + "chart", + "totalRequests", + "totalBlocked", + "passRate" + ], + "title": "UsageOverviewResponse", + "type": "object" + }, + "UsageOverviewRow": { + "properties": { + "avgLatency": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avglatency" + }, + "avgScore": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avgscore" + }, + "failRate": { + "title": "Failrate", + "type": "number" + }, + "id": { + "title": "Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "requestsEvaluated": { + "title": "Requestsevaluated", + "type": "integer" + }, + "status": { + "title": "Status", + "type": "string" + }, + "trend": { + "title": "Trend", + "type": "string" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "name", + "type", + "provider", + "requestsEvaluated", + "failRate", + "avgScore", + "avgLatency", + "status", + "trend" + ], + "title": "UsageOverviewRow", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/apply_guardrail": { + "post": { + "description": "Apply a guardrail to text input and return the processed result.\n\nThis endpoint allows testing guardrails by applying them to custom text inputs.", + "operationId": "apply_guardrail_apply_guardrail_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApplyGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApplyGuardrailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Apply Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails": { + "post": { + "description": "Create a new guardrail\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/guardrails\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"guardrail\": {\n \"guardrail_name\": \"my-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Bedrock content moderation guardrail\"\n }\n }\n }'\n```\n\nExample Response:\n```json\n{\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"my-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Bedrock content moderation guardrail\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n}\n```", + "operationId": "create_guardrail_guardrails_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/apply_guardrail": { + "post": { + "description": "Apply a guardrail to text input and return the processed result.\n\nThis endpoint allows testing guardrails by applying them to custom text inputs.", + "operationId": "apply_guardrail_guardrails_apply_guardrail_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApplyGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApplyGuardrailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Apply Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/list": { + "get": { + "description": "List the guardrails that are available on the proxy server\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/guardrails/list\" -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"guardrails\": [\n {\n \"guardrail_name\": \"bedrock-pre-guard\",\n \"guardrail_info\": {\n \"params\": [\n {\n \"name\": \"toxicity_score\",\n \"type\": \"float\",\n \"description\": \"Score between 0-1 indicating content toxicity level\"\n },\n {\n \"name\": \"pii_detection\",\n \"type\": \"boolean\"\n }\n ]\n }\n }\n ]\n}\n```", + "operationId": "list_guardrails_guardrails_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListGuardrailsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Guardrails", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/register": { + "post": { + "description": "Register a guardrail for onboarding (team submission).\n\nAccepts a guardrail config in the\n[Generic Guardrail API](https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api) format.\nThe submission is stored with status `pending_review` until an admin approves it.", + "operationId": "register_guardrail_guardrails_register_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterGuardrailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Register Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/submissions": { + "get": { + "description": "List team guardrail submissions. Returns only guardrails with a team_id.\n\nAdmins see all submissions. Non-admin users see submissions for teams they are\na member of.\n\nStatus values: pending_review (team-registered, awaiting approval), active (approved), rejected.\n\nOptional filters:\n- status: pending_review | active | rejected\n- team_id: filter by specific team (non-admins must be a member of that team)\n- search: name/description", + "operationId": "list_guardrail_submissions_guardrails_submissions_get", + "parameters": [ + { + "in": "query", + "name": "status", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Status" + } + }, + { + "in": "query", + "name": "team_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + } + }, + { + "in": "query", + "name": "search", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Search" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListGuardrailSubmissionsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Guardrail Submissions", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/submissions/{guardrail_id}": { + "get": { + "description": "Get a single guardrail submission by id. Non-admins may only access submissions for teams they belong to.", + "operationId": "get_guardrail_submission_guardrails_submissions__guardrail_id__get", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GuardrailSubmissionItem" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Guardrail Submission", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/submissions/{guardrail_id}/approve": { + "post": { + "description": "Approve a pending guardrail submission: set status to active and initialize in memory (admin only).", + "operationId": "approve_guardrail_submission_guardrails_submissions__guardrail_id__approve_post", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Approve Guardrail Submission", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/submissions/{guardrail_id}/reject": { + "post": { + "description": "Reject a guardrail submission (admin only).", + "operationId": "reject_guardrail_submission_guardrails_submissions__guardrail_id__reject_post", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Reject Guardrail Submission", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/test_custom_code": { + "post": { + "description": "Test custom code guardrail logic without creating a guardrail.\n\nThis endpoint allows admins to experiment with custom code guardrails by:\n1. Compiling the provided code in a sandbox\n2. Executing the apply_guardrail function with test input\n3. Returning the result (allow/block/modify)\n\n\ud83d\udc49 [Custom Code Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/custom_code_guardrail)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/guardrails/test_custom_code\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"custom_code\": \"def apply_guardrail(inputs, request_data, input_type):\\n for text in inputs[\\\"texts\\\"]:\\n if regex_match(text, r\\\"\\\\d{3}-\\\\d{2}-\\\\d{4}\\\"):\\n return block(\\\"SSN detected\\\")\\n return allow()\",\n \"test_input\": {\n \"texts\": [\"My SSN is 123-45-6789\"]\n },\n \"input_type\": \"request\"\n }'\n```\n\nExample Success Response (blocked):\n```json\n{\n \"success\": true,\n \"result\": {\n \"action\": \"block\",\n \"reason\": \"SSN detected\"\n },\n \"error\": null,\n \"error_type\": null\n}\n```\n\nExample Success Response (allowed):\n```json\n{\n \"success\": true,\n \"result\": {\n \"action\": \"allow\"\n },\n \"error\": null,\n \"error_type\": null\n}\n```\n\nExample Success Response (modified):\n```json\n{\n \"success\": true,\n \"result\": {\n \"action\": \"modify\",\n \"texts\": [\"My SSN is [REDACTED]\"]\n },\n \"error\": null,\n \"error_type\": null\n}\n```\n\nExample Error Response (compilation error):\n```json\n{\n \"success\": false,\n \"result\": null,\n \"error\": \"Syntax error in custom code: invalid syntax (, line 1)\",\n \"error_type\": \"compilation\"\n}\n```", + "operationId": "test_custom_code_guardrail_guardrails_test_custom_code_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestCustomCodeGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestCustomCodeGuardrailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Custom Code Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/ui/add_guardrail_settings": { + "get": { + "description": "Get the UI settings for the guardrails\n\nReturns:\n- Supported entities for guardrails\n- Supported modes for guardrails\n- PII entity categories for UI organization\n- Content filter settings (patterns and categories)", + "operationId": "get_guardrail_ui_settings_guardrails_ui_add_guardrail_settings_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Guardrail Ui Settings", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/ui/category_yaml/{category_name}": { + "get": { + "description": "Get the YAML or JSON content for a specific content filter category.\n\nArgs:\n category_name: The name of the category (e.g., \"bias_gender\", \"harmful_self_harm\")\n\nReturns:\n The raw YAML or JSON content of the category file with file type indicator", + "operationId": "get_category_yaml_guardrails_ui_category_yaml__category_name__get", + "parameters": [ + { + "in": "path", + "name": "category_name", + "required": true, + "schema": { + "title": "Category Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Category Yaml", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/ui/major_airlines": { + "get": { + "description": "Get the major airlines list from IATA (competitor intent, airline type).\nReturns airline id, match variants (pipe-separated), and tags.", + "operationId": "get_major_airlines_guardrails_ui_major_airlines_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Major Airlines", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/ui/provider_specific_params": { + "get": { + "description": "Get provider-specific parameters for different guardrail types.\n\nReturns a dictionary mapping guardrail providers to their specific parameters,\nincluding parameter names, descriptions, and whether they are required.\n\nExample Response:\n```json\n{\n \"bedrock\": {\n \"guardrailIdentifier\": {\n \"description\": \"The ID of your guardrail on Bedrock\",\n \"required\": true,\n \"type\": null\n },\n \"guardrailVersion\": {\n \"description\": \"The version of your Bedrock guardrail (e.g., DRAFT or version number)\",\n \"required\": true,\n \"type\": null\n }\n },\n \"azure_content_safety_text_moderation\": {\n \"api_key\": {\n \"description\": \"API key for the Azure Content Safety Text Moderation guardrail\",\n \"required\": false,\n \"type\": null\n },\n \"optional_params\": {\n \"description\": \"Optional parameters for the Azure Content Safety Text Moderation guardrail\",\n \"required\": true,\n \"type\": \"nested\",\n \"fields\": {\n \"severity_threshold\": {\n \"description\": \"Severity threshold for the Azure Content Safety Text Moderation guardrail across all categories\",\n \"required\": false,\n \"type\": null\n },\n \"categories\": {\n \"description\": \"Categories to scan for the Azure Content Safety Text Moderation guardrail\",\n \"required\": false,\n \"type\": \"multiselect\",\n \"options\": [\"Hate\", \"SelfHarm\", \"Sexual\", \"Violence\"],\n \"default_value\": None\n }\n }\n }\n }\n}\n```", + "operationId": "get_provider_specific_params_guardrails_ui_provider_specific_params_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Provider Specific Params", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/usage/detail/{guardrail_id}": { + "get": { + "description": "Return single guardrail usage metrics and time series.", + "operationId": "guardrails_usage_detail_guardrails_usage_detail__guardrail_id__get", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + }, + { + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Start Date" + } + }, + { + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageDetailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Guardrails Usage Detail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/usage/logs": { + "get": { + "description": "Return paginated run logs for a guardrail (or policy) from SpendLogs via index.", + "operationId": "guardrails_usage_logs_guardrails_usage_logs_get", + "parameters": [ + { + "in": "query", + "name": "guardrail_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Guardrail Id" + } + }, + { + "in": "query", + "name": "policy_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Policy Id" + } + }, + { + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "minimum": 1, + "title": "Page", + "type": "integer" + } + }, + { + "in": "query", + "name": "page_size", + "required": false, + "schema": { + "default": 50, + "maximum": 100, + "minimum": 1, + "title": "Page Size", + "type": "integer" + } + }, + { + "in": "query", + "name": "action", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Action" + } + }, + { + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Start Date" + } + }, + { + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageLogsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Guardrails Usage Logs", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/usage/overview": { + "get": { + "description": "Return guardrail performance overview for the dashboard.", + "operationId": "guardrails_usage_overview_guardrails_usage_overview_get", + "parameters": [ + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "Start Date" + } + }, + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageOverviewResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Guardrails Usage Overview", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/validate_blocked_words_file": { + "post": { + "description": "Validate a blocked_words YAML file content.\n\nArgs:\n request: Dictionary with 'file_content' key containing the YAML string\n\nReturns:\n Dictionary with 'valid' boolean and either 'message'/'errors' depending on result\n\nExample Request:\n```json\n{\n \"file_content\": \"blocked_words:\\n - keyword: \\\"test\\\"\\n action: \\\"BLOCK\\\"\"\n}\n```\n\nExample Success Response:\n```json\n{\n \"valid\": true,\n \"message\": \"Valid YAML file with 2 blocked words\"\n}\n```\n\nExample Error Response:\n```json\n{\n \"valid\": false,\n \"errors\": [\"Entry 0: missing 'action' field\"]\n}\n```", + "operationId": "validate_blocked_words_file_guardrails_validate_blocked_words_file_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "string" + }, + "title": "Request", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Validate Blocked Words File", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/{guardrail_id}": { + "delete": { + "description": "Delete a guardrail\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/guardrails/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Guardrail 123e4567-e89b-12d3-a456-426614174000 deleted successfully\"\n}\n```", + "operationId": "delete_guardrail_guardrails__guardrail_id__delete", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Guardrail", + "tags": [ + "guardrails" + ] + }, + "get": { + "description": "Get detailed information about a specific guardrail by ID\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/guardrails/123e4567-e89b-12d3-a456-426614174000/info\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"my-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Bedrock content moderation guardrail\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n}\n```", + "operationId": "get_guardrail_info_guardrails__guardrail_id__get", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Guardrail Info", + "tags": [ + "guardrails" + ] + }, + "patch": { + "description": "Partially update an existing guardrail\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nThis endpoint allows updating specific fields of a guardrail without sending the entire object.\nOnly the following fields can be updated:\n- guardrail_name: The name of the guardrail\n- default_on: Whether the guardrail is enabled by default\n- guardrail_info: Additional information about the guardrail\n\nExample Request:\n```bash\ncurl -X PATCH \"http://localhost:4000/guardrails/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"guardrail_name\": \"updated-name\",\n \"default_on\": true,\n \"guardrail_info\": {\n \"description\": \"Updated description\"\n }\n }'\n```\n\nExample Response:\n```json\n{\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"updated-name\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Updated description\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T14:22:33.456Z\"\n}\n```", + "operationId": "patch_guardrail_guardrails__guardrail_id__patch", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PatchGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Patch Guardrail", + "tags": [ + "guardrails" + ] + }, + "put": { + "description": "Update an existing guardrail\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/guardrails/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"guardrail\": {\n \"guardrail_name\": \"updated-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"1.0\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Updated Bedrock content moderation guardrail\"\n }\n }\n }'\n```\n\nExample Response:\n```json\n{\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"updated-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"1.0\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Updated Bedrock content moderation guardrail\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T13:45:12.345Z\"\n}\n```", + "operationId": "update_guardrail_guardrails__guardrail_id__put", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateGuardrailRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Guardrail", + "tags": [ + "guardrails" + ] + } + }, + "/guardrails/{guardrail_id}/info": { + "get": { + "description": "Get detailed information about a specific guardrail by ID\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/guardrails/123e4567-e89b-12d3-a456-426614174000/info\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"my-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Bedrock content moderation guardrail\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n}\n```", + "operationId": "get_guardrail_info_guardrails__guardrail_id__info_get", + "parameters": [ + { + "in": "path", + "name": "guardrail_id", + "required": true, + "schema": { + "title": "Guardrail Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Guardrail Info", + "tags": [ + "guardrails" + ] + } + }, + "/policies/usage/overview": { + "get": { + "description": "Return policy performance overview for the dashboard.", + "operationId": "policies_usage_overview_policies_usage_overview_get", + "parameters": [ + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "Start Date" + } + }, + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageOverviewResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Policies Usage Overview", + "tags": [ + "guardrails" + ] + } + }, + "/v2/guardrails/list": { + "get": { + "description": "List the guardrails that are available in the database using GuardrailRegistry\n\n\ud83d\udc49 [Guardrail docs](https://docs.litellm.ai/docs/proxy/guardrails/quick_start)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/v2/guardrails/list\" -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"guardrails\": [\n {\n \"guardrail_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"guardrail_name\": \"my-bedrock-guard\",\n \"litellm_params\": {\n \"guardrail\": \"bedrock\",\n \"mode\": \"pre_call\",\n \"guardrailIdentifier\": \"ff6ujrregl1q\",\n \"guardrailVersion\": \"DRAFT\",\n \"default_on\": true\n },\n \"guardrail_info\": {\n \"description\": \"Bedrock content moderation guardrail\"\n }\n }\n ]\n}\n```", + "operationId": "list_guardrails_v2_v2_guardrails_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListGuardrailsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Guardrails V2", + "tags": [ + "guardrails" + ] + } + } + } + }, + "jwt_mappings": { + "components": { + "schemas": { + "CreateJWTKeyMappingRequest": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "jwt_claim_name": { + "title": "Jwt Claim Name", + "type": "string" + }, + "jwt_claim_value": { + "title": "Jwt Claim Value", + "type": "string" + }, + "key": { + "title": "Key", + "type": "string" + } + }, + "required": [ + "jwt_claim_name", + "jwt_claim_value", + "key" + ], + "title": "CreateJWTKeyMappingRequest", + "type": "object" + }, + "DeleteJWTKeyMappingRequest": { + "properties": { + "id": { + "title": "Id", + "type": "string" + } + }, + "required": [ + "id" + ], + "title": "DeleteJWTKeyMappingRequest", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "JWTKeyMappingResponse": { + "properties": { + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "id": { + "title": "Id", + "type": "string" + }, + "is_active": { + "title": "Is Active", + "type": "boolean" + }, + "jwt_claim_name": { + "title": "Jwt Claim Name", + "type": "string" + }, + "jwt_claim_value": { + "title": "Jwt Claim Value", + "type": "string" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated By" + } + }, + "required": [ + "id", + "jwt_claim_name", + "jwt_claim_value", + "is_active", + "created_at", + "updated_at" + ], + "title": "JWTKeyMappingResponse", + "type": "object" + }, + "UpdateJWTKeyMappingRequest": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "id": { + "title": "Id", + "type": "string" + }, + "is_active": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Is Active" + }, + "key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key" + } + }, + "required": [ + "id" + ], + "title": "UpdateJWTKeyMappingRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/jwt/key/mapping/delete": { + "post": { + "operationId": "delete_jwt_key_mapping_jwt_key_mapping_delete_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteJWTKeyMappingRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Jwt Key Mapping", + "tags": [ + "jwt_mappings" + ] + } + }, + "/jwt/key/mapping/info": { + "get": { + "operationId": "info_jwt_key_mapping_jwt_key_mapping_info_get", + "parameters": [ + { + "in": "query", + "name": "id", + "required": true, + "schema": { + "title": "Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JWTKeyMappingResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Info Jwt Key Mapping", + "tags": [ + "jwt_mappings" + ] + } + }, + "/jwt/key/mapping/list": { + "get": { + "operationId": "list_jwt_key_mappings_jwt_key_mapping_list_get", + "parameters": [ + { + "description": "Page number", + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "description": "Page number", + "minimum": 1, + "title": "Page", + "type": "integer" + } + }, + { + "description": "Page size", + "in": "query", + "name": "size", + "required": false, + "schema": { + "default": 50, + "description": "Page size", + "maximum": 100, + "minimum": 1, + "title": "Size", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Jwt Key Mappings", + "tags": [ + "jwt_mappings" + ] + } + }, + "/jwt/key/mapping/new": { + "post": { + "operationId": "create_jwt_key_mapping_jwt_key_mapping_new_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateJWTKeyMappingRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JWTKeyMappingResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Jwt Key Mapping", + "tags": [ + "jwt_mappings" + ] + } + }, + "/jwt/key/mapping/update": { + "post": { + "operationId": "update_jwt_key_mapping_jwt_key_mapping_update_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateJWTKeyMappingRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JWTKeyMappingResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Jwt Key Mapping", + "tags": [ + "jwt_mappings" + ] + } + } + } + }, + "langfuse_passthrough": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/langfuse/{endpoint}": { + "delete": { + "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", + "operationId": "langfuse_proxy_route_langfuse__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Langfuse Proxy Route", + "tags": [ + "langfuse_passthrough" + ] + }, + "get": { + "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", + "operationId": "langfuse_proxy_route_langfuse__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Langfuse Proxy Route", + "tags": [ + "langfuse_passthrough" + ] + }, + "patch": { + "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", + "operationId": "langfuse_proxy_route_langfuse__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Langfuse Proxy Route", + "tags": [ + "langfuse_passthrough" + ] + }, + "post": { + "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", + "operationId": "langfuse_proxy_route_langfuse__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Langfuse Proxy Route", + "tags": [ + "langfuse_passthrough" + ] + }, + "put": { + "description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)", + "operationId": "langfuse_proxy_route_langfuse__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Langfuse Proxy Route", + "tags": [ + "langfuse_passthrough" + ] + } + } + } + }, + "mcp_app": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "MCPCredentials": { + "properties": { + "auth_value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Value" + }, + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Access Key Id" + }, + "aws_region_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Region Name" + }, + "aws_role_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Role Name" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Secret Access Key" + }, + "aws_service_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Service Name" + }, + "aws_session_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Name" + }, + "aws_session_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Token" + }, + "client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Id" + }, + "client_secret": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Secret" + }, + "scopes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Scopes" + } + }, + "title": "MCPCredentials", + "type": "object" + }, + "NewMCPServerRequest": { + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "allow_all_keys": { + "default": false, + "title": "Allow All Keys", + "type": "boolean" + }, + "allowed_tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Allowed Tools" + }, + "approval_status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Approval Status" + }, + "args": { + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "auth_type": { + "anyOf": [ + { + "enum": [ + "none", + "api_key", + "bearer_token", + "basic", + "authorization", + "oauth2", + "aws_sigv4", + "token" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Type" + }, + "authorization_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization Url" + }, + "available_on_public_internet": { + "default": true, + "title": "Available On Public Internet", + "type": "boolean" + }, + "byok_api_key_help_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Byok Api Key Help Url" + }, + "byok_description": { + "items": { + "type": "string" + }, + "title": "Byok Description", + "type": "array" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Command" + }, + "credentials": { + "anyOf": [ + { + "$ref": "#/components/schemas/MCPCredentials" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "is_byok": { + "default": false, + "title": "Is Byok", + "type": "boolean" + }, + "mcp_access_groups": { + "items": { + "type": "string" + }, + "title": "Mcp Access Groups", + "type": "array" + }, + "mcp_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Info" + }, + "oauth2_flow": { + "anyOf": [ + { + "enum": [ + "client_credentials", + "authorization_code" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Oauth2 Flow" + }, + "registration_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Registration Url" + }, + "server_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Id" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Url" + }, + "spec_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Spec Path" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted At" + }, + "submitted_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted By" + }, + "token_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token Url" + }, + "tool_name_to_description": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Description" + }, + "tool_name_to_display_name": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Display Name" + }, + "transport": { + "default": "sse", + "enum": [ + "sse", + "http", + "stdio" + ], + "title": "Transport", + "type": "string" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + } + }, + "title": "NewMCPServerRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/mcp-rest/test/connection": { + "post": { + "description": "Test if we can connect to the provided MCP server before adding it", + "operationId": "test_connection_mcp_rest_test_connection_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Connection", + "tags": [ + "mcp_app" + ] + } + }, + "/mcp-rest/test/tools/list": { + "post": { + "description": "Preview tools available from MCP server before adding it", + "operationId": "test_tools_list_mcp_rest_test_tools_list_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Tools List", + "tags": [ + "mcp_app" + ] + } + }, + "/mcp-rest/tools/call": { + "post": { + "description": "REST API to call a specific MCP tool with the provided arguments", + "operationId": "call_tool_rest_api_mcp_rest_tools_call_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Call Tool Rest Api", + "tags": [ + "mcp_app" + ] + } + }, + "/mcp-rest/tools/list": { + "get": { + "description": "List all available tools with information about the server they belong to.\n\nExample response:\n{\n \"tools\": [\n {\n \"name\": \"create_zap\",\n \"description\": \"Create a new zap\",\n \"inputSchema\": \"tool_input_schema\",\n \"mcp_info\": {\n \"server_name\": \"zapier\",\n \"logo_url\": \"https://www.zapier.com/logo.png\",\n }\n }\n ],\n \"error\": null,\n \"message\": \"Successfully retrieved tools\"\n}", + "operationId": "list_tool_rest_api_mcp_rest_tools_list_get", + "parameters": [ + { + "description": "The server id to list tools for", + "in": "query", + "name": "server_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The server id to list tools for", + "title": "Server Id" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response List Tool Rest Api Mcp Rest Tools List Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Tool Rest Api", + "tags": [ + "mcp_app" + ] + } + } + } + }, + "mcp_byok_oauth": { + "components": { + "schemas": {} + }, + "paths": {} + }, + "mcp_discoverable": { + "components": { + "schemas": { + "CallbacksByType": { + "properties": { + "failure": { + "items": { + "type": "string" + }, + "title": "Failure", + "type": "array" + }, + "success": { + "items": { + "type": "string" + }, + "title": "Success", + "type": "array" + }, + "success_and_failure": { + "items": { + "type": "string" + }, + "title": "Success And Failure", + "type": "array" + } + }, + "required": [ + "success", + "failure", + "success_and_failure" + ], + "title": "CallbacksByType", + "type": "object" + } + } + }, + "paths": { + "/callbacks/configs": { + "get": { + "description": "Get Available Callback Configurations\n\nReturns the configuration details for all available logging callbacks,\nincluding supported parameters, field types, and descriptions.", + "operationId": "get_callback_configs_callbacks_configs_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Callback Configs", + "tags": [ + "mcp_discoverable" + ] + } + }, + "/callbacks/list": { + "get": { + "description": "View List of Active Logging Callbacks", + "operationId": "list_callbacks_callbacks_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CallbacksByType" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Callbacks", + "tags": [ + "mcp_discoverable" + ] + } + } + } + }, + "mcp_management": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "LiteLLM_MCPServerTable": { + "description": "Represents a LiteLLM_MCPServerTable record", + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "allow_all_keys": { + "default": false, + "title": "Allow All Keys", + "type": "boolean" + }, + "allowed_tools": { + "items": { + "type": "string" + }, + "title": "Allowed Tools", + "type": "array" + }, + "approval_status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "active", + "description": "Approval status: 'pending_review', 'active', 'rejected'", + "title": "Approval Status" + }, + "args": { + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "auth_type": { + "anyOf": [ + { + "enum": [ + "none", + "api_key", + "bearer_token", + "basic", + "authorization", + "oauth2", + "aws_sigv4", + "token" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Type" + }, + "authorization_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization Url" + }, + "available_on_public_internet": { + "default": true, + "title": "Available On Public Internet", + "type": "boolean" + }, + "byok_api_key_help_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Byok Api Key Help Url" + }, + "byok_description": { + "items": { + "type": "string" + }, + "title": "Byok Description", + "type": "array" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Command" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "credentials": { + "anyOf": [ + { + "$ref": "#/components/schemas/MCPCredentials" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "extra_headers": { + "items": { + "type": "string" + }, + "title": "Extra Headers", + "type": "array" + }, + "has_user_credential": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Has User Credential" + }, + "health_check_error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Health Check Error" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "is_byok": { + "default": false, + "title": "Is Byok", + "type": "boolean" + }, + "last_health_check": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Health Check" + }, + "mcp_access_groups": { + "items": { + "type": "string" + }, + "title": "Mcp Access Groups", + "type": "array" + }, + "mcp_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Info" + }, + "registration_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Registration Url" + }, + "review_notes": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Review Notes" + }, + "reviewed_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reviewed At" + }, + "server_id": { + "title": "Server Id", + "type": "string" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Url" + }, + "spec_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Spec Path" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "status": { + "anyOf": [ + { + "enum": [ + "healthy", + "unhealthy", + "unknown" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": "unknown", + "description": "Health status: 'healthy', 'unhealthy', 'unknown'", + "title": "Status" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted At" + }, + "submitted_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Submitted By" + }, + "teams": { + "items": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "object" + }, + "title": "Teams", + "type": "array" + }, + "token_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token Url" + }, + "tool_name_to_description": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Description" + }, + "tool_name_to_display_name": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Display Name" + }, + "transport": { + "enum": [ + "sse", + "http", + "stdio" + ], + "title": "Transport", + "type": "string" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated By" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + } + }, + "required": [ + "server_id", + "transport" + ], + "title": "LiteLLM_MCPServerTable", + "type": "object" + }, + "MCPCredentials": { + "properties": { + "auth_value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Value" + }, + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Access Key Id" + }, + "aws_region_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Region Name" + }, + "aws_role_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Role Name" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Secret Access Key" + }, + "aws_service_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Service Name" + }, + "aws_session_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Name" + }, + "aws_session_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Token" + }, + "client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Id" + }, + "client_secret": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Secret" + }, + "scopes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Scopes" + } + }, + "title": "MCPCredentials", + "type": "object" + }, + "MCPOAuthUserCredentialRequest": { + "description": "Stores a user's OAuth2 token for an OpenAPI MCP server.", + "properties": { + "access_token": { + "title": "Access Token", + "type": "string" + }, + "expires_in": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Expires In" + }, + "refresh_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Refresh Token" + }, + "scopes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Scopes" + } + }, + "required": [ + "access_token" + ], + "title": "MCPOAuthUserCredentialRequest", + "type": "object" + }, + "MCPOAuthUserCredentialStatus": { + "description": "Describes whether the calling user has a stored OAuth credential.", + "properties": { + "connected_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Connected At" + }, + "expires_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Expires At" + }, + "has_credential": { + "title": "Has Credential", + "type": "boolean" + }, + "is_expired": { + "default": false, + "title": "Is Expired", + "type": "boolean" + }, + "server_id": { + "title": "Server Id", + "type": "string" + } + }, + "required": [ + "server_id", + "has_credential" + ], + "title": "MCPOAuthUserCredentialStatus", + "type": "object" + }, + "MCPSubmissionsSummary": { + "properties": { + "active": { + "title": "Active", + "type": "integer" + }, + "items": { + "items": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + }, + "title": "Items", + "type": "array" + }, + "pending_review": { + "title": "Pending Review", + "type": "integer" + }, + "rejected": { + "title": "Rejected", + "type": "integer" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "total", + "pending_review", + "active", + "rejected", + "items" + ], + "title": "MCPSubmissionsSummary", + "type": "object" + }, + "MCPToolsetTool": { + "properties": { + "server_id": { + "title": "Server Id", + "type": "string" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + } + }, + "required": [ + "server_id", + "tool_name" + ], + "title": "MCPToolsetTool", + "type": "object" + }, + "MCPUserCredentialListItem": { + "description": "One entry in the /user-credentials list.", + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "connected_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Connected At" + }, + "credential_type": { + "title": "Credential Type", + "type": "string" + }, + "expires_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Expires At" + }, + "has_credential": { + "title": "Has Credential", + "type": "boolean" + }, + "server_id": { + "title": "Server Id", + "type": "string" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + } + }, + "required": [ + "server_id", + "credential_type", + "has_credential" + ], + "title": "MCPUserCredentialListItem", + "type": "object" + }, + "MCPUserCredentialRequest": { + "properties": { + "credential": { + "title": "Credential", + "type": "string" + }, + "save": { + "default": true, + "title": "Save", + "type": "boolean" + } + }, + "required": [ + "credential" + ], + "title": "MCPUserCredentialRequest", + "type": "object" + }, + "MCPUserCredentialResponse": { + "properties": { + "has_credential": { + "title": "Has Credential", + "type": "boolean" + }, + "server_id": { + "title": "Server Id", + "type": "string" + } + }, + "required": [ + "server_id", + "has_credential" + ], + "title": "MCPUserCredentialResponse", + "type": "object" + }, + "MakeMCPServersPublicRequest": { + "properties": { + "mcp_server_ids": { + "items": { + "type": "string" + }, + "title": "Mcp Server Ids", + "type": "array" + } + }, + "required": [ + "mcp_server_ids" + ], + "title": "MakeMCPServersPublicRequest", + "type": "object" + }, + "NewMCPServerRequest": { + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "allow_all_keys": { + "default": false, + "title": "Allow All Keys", + "type": "boolean" + }, + "allowed_tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Allowed Tools" + }, + "approval_status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Approval Status" + }, + "args": { + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "auth_type": { + "anyOf": [ + { + "enum": [ + "none", + "api_key", + "bearer_token", + "basic", + "authorization", + "oauth2", + "aws_sigv4", + "token" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Type" + }, + "authorization_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization Url" + }, + "available_on_public_internet": { + "default": true, + "title": "Available On Public Internet", + "type": "boolean" + }, + "byok_api_key_help_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Byok Api Key Help Url" + }, + "byok_description": { + "items": { + "type": "string" + }, + "title": "Byok Description", + "type": "array" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Command" + }, + "credentials": { + "anyOf": [ + { + "$ref": "#/components/schemas/MCPCredentials" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "is_byok": { + "default": false, + "title": "Is Byok", + "type": "boolean" + }, + "mcp_access_groups": { + "items": { + "type": "string" + }, + "title": "Mcp Access Groups", + "type": "array" + }, + "mcp_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Info" + }, + "oauth2_flow": { + "anyOf": [ + { + "enum": [ + "client_credentials", + "authorization_code" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Oauth2 Flow" + }, + "registration_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Registration Url" + }, + "server_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Id" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Url" + }, + "spec_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Spec Path" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted At" + }, + "submitted_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted By" + }, + "token_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token Url" + }, + "tool_name_to_description": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Description" + }, + "tool_name_to_display_name": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Display Name" + }, + "transport": { + "default": "sse", + "enum": [ + "sse", + "http", + "stdio" + ], + "title": "Transport", + "type": "string" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + } + }, + "title": "NewMCPServerRequest", + "type": "object" + }, + "NewMCPToolsetRequest": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "tools": { + "default": [], + "items": { + "$ref": "#/components/schemas/MCPToolsetTool" + }, + "title": "Tools", + "type": "array" + }, + "toolset_name": { + "title": "Toolset Name", + "type": "string" + } + }, + "required": [ + "toolset_name" + ], + "title": "NewMCPToolsetRequest", + "type": "object" + }, + "RejectMCPServerRequest": { + "properties": { + "review_notes": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Review Notes" + } + }, + "title": "RejectMCPServerRequest", + "type": "object" + }, + "UpdateMCPServerRequest": { + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "allow_all_keys": { + "default": false, + "title": "Allow All Keys", + "type": "boolean" + }, + "allowed_tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Allowed Tools" + }, + "args": { + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "auth_type": { + "anyOf": [ + { + "enum": [ + "none", + "api_key", + "bearer_token", + "basic", + "authorization", + "oauth2", + "aws_sigv4", + "token" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Type" + }, + "authorization_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization Url" + }, + "available_on_public_internet": { + "default": true, + "title": "Available On Public Internet", + "type": "boolean" + }, + "byok_api_key_help_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Byok Api Key Help Url" + }, + "byok_description": { + "items": { + "type": "string" + }, + "title": "Byok Description", + "type": "array" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Command" + }, + "credentials": { + "anyOf": [ + { + "$ref": "#/components/schemas/MCPCredentials" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "is_byok": { + "default": false, + "title": "Is Byok", + "type": "boolean" + }, + "mcp_access_groups": { + "items": { + "type": "string" + }, + "title": "Mcp Access Groups", + "type": "array" + }, + "mcp_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Info" + }, + "registration_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Registration Url" + }, + "server_id": { + "title": "Server Id", + "type": "string" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Url" + }, + "spec_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Spec Path" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "token_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token Url" + }, + "tool_name_to_description": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Description" + }, + "tool_name_to_display_name": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Display Name" + }, + "transport": { + "default": "sse", + "enum": [ + "sse", + "http", + "stdio" + ], + "title": "Transport", + "type": "string" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + } + }, + "required": [ + "server_id" + ], + "title": "UpdateMCPServerRequest", + "type": "object" + }, + "UpdateMCPToolsetRequest": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "tools": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/MCPToolsetTool" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "toolset_id": { + "title": "Toolset Id", + "type": "string" + }, + "toolset_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Toolset Name" + } + }, + "required": [ + "toolset_id" + ], + "title": "UpdateMCPToolsetRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/v1/mcp/access_groups": { + "get": { + "description": "Get all available MCP access groups from the database AND config", + "operationId": "get_mcp_access_groups_v1_mcp_access_groups_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Mcp Access Groups", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/discover": { + "get": { + "description": "Returns a curated list of well-known MCP servers for discovery UI", + "operationId": "discover_mcp_servers_v1_mcp_discover_get", + "parameters": [ + { + "description": "Search filter for server names and descriptions", + "in": "query", + "name": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Search filter for server names and descriptions", + "title": "Query" + } + }, + { + "description": "Filter by category", + "in": "query", + "name": "category", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Filter by category", + "title": "Category" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Discover Mcp Servers", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/make_public": { + "post": { + "description": "Allows making MCP servers public for AI Hub", + "operationId": "make_mcp_servers_public_v1_mcp_make_public_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MakeMCPServersPublicRequest" + } + } + }, + "required": true + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Make Mcp Servers Public", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/network/client-ip": { + "get": { + "description": "Returns the caller's IP address as seen by the proxy.", + "operationId": "get_client_ip_v1_mcp_network_client_ip_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Client Ip", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/openapi-registry": { + "get": { + "description": "Returns well-known OpenAPI APIs with OAuth 2.0 metadata for the OpenAPI MCP picker", + "operationId": "get_openapi_registry_v1_mcp_openapi_registry_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Openapi Registry", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/registry.json": { + "get": { + "description": "MCP registry endpoint. Spec: https://github.com/modelcontextprotocol/registry", + "operationId": "get_mcp_registry_v1_mcp_registry_json_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Get Mcp Registry", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server": { + "get": { + "description": "Returns the mcp server list with associated teams", + "operationId": "fetch_all_mcp_servers_v1_mcp_server_get", + "parameters": [ + { + "description": "Filter MCP servers by team scope. When provided, returns only servers the team has access to plus globally available (allow_all_keys) servers. Used by the Create Key UI to show team-scoped MCP servers.", + "in": "query", + "name": "team_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Filter MCP servers by team scope. When provided, returns only servers the team has access to plus globally available (allow_all_keys) servers. Used by the Create Key UI to show team-scoped MCP servers.", + "title": "Team Id" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + }, + "title": "Response Fetch All Mcp Servers V1 Mcp Server Get", + "type": "array" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fetch All Mcp Servers", + "tags": [ + "mcp_management" + ] + }, + "post": { + "description": "Allows creation of mcp servers", + "operationId": "add_mcp_server_v1_mcp_server_post", + "parameters": [ + { + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "title": "Litellm-Changed-By" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Add Mcp Server", + "tags": [ + "mcp_management" + ] + }, + "put": { + "description": "Allows deleting mcp serves in the db", + "operationId": "edit_mcp_server_v1_mcp_server_put", + "parameters": [ + { + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "title": "Litellm-Changed-By" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Edit Mcp Server", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/health": { + "get": { + "description": "Health check for MCP servers", + "operationId": "health_check_servers_v1_mcp_server_health_get", + "parameters": [ + { + "description": "Server IDs to check. If not provided, checks all accessible servers.", + "in": "query", + "name": "server_ids", + "required": false, + "schema": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Server IDs to check. If not provided, checks all accessible servers.", + "title": "Server Ids" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Health Check Servers", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/oauth/session": { + "post": { + "description": "Temporarily cache an MCP server in memory without writing to the database", + "operationId": "add_session_mcp_server_v1_mcp_server_oauth_session_post", + "parameters": [ + { + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "title": "Litellm-Changed-By" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Add Session Mcp Server", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/register": { + "post": { + "description": "Submit a new MCP server for admin review (non-admin users). Mirrors POST /guardrails/register.", + "operationId": "register_mcp_server_v1_mcp_server_register_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Register Mcp Server", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/submissions": { + "get": { + "description": "Returns all MCP servers submitted by non-admin users (admin review queue). Mirrors GET /guardrails/submissions.", + "operationId": "get_mcp_server_submissions_v1_mcp_server_submissions_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPSubmissionsSummary" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Mcp Server Submissions", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}": { + "delete": { + "description": "Allows deleting mcp serves in the db", + "operationId": "remove_mcp_server_v1_mcp_server__server_id__delete", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + }, + { + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + "title": "Litellm-Changed-By" + } + } + ], + "responses": { + "202": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Remove Mcp Server", + "tags": [ + "mcp_management" + ] + }, + "get": { + "description": "Returns the mcp server info", + "operationId": "fetch_mcp_server_v1_mcp_server__server_id__get", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fetch Mcp Server", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}/approve": { + "put": { + "description": "Approve a pending MCP server submission (admin only). Mirrors PUT /guardrails/{id}/approve.", + "operationId": "approve_mcp_server_submission_v1_mcp_server__server_id__approve_put", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Approve Mcp Server Submission", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}/oauth-user-credential": { + "delete": { + "description": "Revoke the calling user's stored OAuth2 token for an MCP server", + "operationId": "delete_mcp_oauth_user_credential_v1_mcp_server__server_id__oauth_user_credential_delete", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPOAuthUserCredentialStatus" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Mcp Oauth User Credential", + "tags": [ + "mcp_management" + ] + }, + "post": { + "description": "Store the calling user's OAuth2 token for an OpenAPI MCP server", + "operationId": "store_mcp_oauth_user_credential_v1_mcp_server__server_id__oauth_user_credential_post", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPOAuthUserCredentialRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPOAuthUserCredentialStatus" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Store Mcp Oauth User Credential", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}/oauth-user-credential/status": { + "get": { + "description": "Check whether the calling user has a stored OAuth2 credential for this MCP server", + "operationId": "get_mcp_oauth_user_credential_status_v1_mcp_server__server_id__oauth_user_credential_status_get", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPOAuthUserCredentialStatus" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Mcp Oauth User Credential Status", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}/reject": { + "put": { + "description": "Reject a pending MCP server submission (admin only). Mirrors PUT /guardrails/{id}/reject.", + "operationId": "reject_mcp_server_submission_v1_mcp_server__server_id__reject_put", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RejectMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_MCPServerTable" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Reject Mcp Server Submission", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/server/{server_id}/user-credential": { + "delete": { + "description": "Delete the calling user's stored API key for a BYOK MCP server", + "operationId": "delete_mcp_user_credential_v1_mcp_server__server_id__user_credential_delete", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPUserCredentialResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Mcp User Credential", + "tags": [ + "mcp_management" + ] + }, + "post": { + "description": "Store or update the calling user's API key for a BYOK MCP server", + "operationId": "store_mcp_user_credential_v1_mcp_server__server_id__user_credential_post", + "parameters": [ + { + "in": "path", + "name": "server_id", + "required": true, + "schema": { + "title": "Server Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPUserCredentialRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MCPUserCredentialResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Store Mcp User Credential", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/tools": { + "get": { + "description": "Get all MCP tools available for the current key, including those from access groups", + "operationId": "get_mcp_tools_v1_mcp_tools_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Mcp Tools", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/toolset": { + "get": { + "description": "List MCP toolsets accessible to the calling key", + "operationId": "fetch_mcp_toolsets_v1_mcp_toolset_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fetch Mcp Toolsets", + "tags": [ + "mcp_management" + ] + }, + "post": { + "description": "Create a new MCP toolset (admin only)", + "operationId": "add_mcp_toolset_v1_mcp_toolset_post", + "parameters": [ + { + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Litellm-Changed-By" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPToolsetRequest" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Add Mcp Toolset", + "tags": [ + "mcp_management" + ] + }, + "put": { + "description": "Update an existing MCP toolset (admin only)", + "operationId": "edit_mcp_toolset_v1_mcp_toolset_put", + "parameters": [ + { + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Litellm-Changed-By" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateMCPToolsetRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Edit Mcp Toolset", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/toolset/{toolset_id}": { + "delete": { + "description": "Delete an MCP toolset (admin only)", + "operationId": "remove_mcp_toolset_v1_mcp_toolset__toolset_id__delete", + "parameters": [ + { + "in": "path", + "name": "toolset_id", + "required": true, + "schema": { + "title": "Toolset Id", + "type": "string" + } + }, + { + "in": "header", + "name": "litellm-changed-by", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Litellm-Changed-By" + } + } + ], + "responses": { + "202": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Remove Mcp Toolset", + "tags": [ + "mcp_management" + ] + }, + "get": { + "description": "Get a specific MCP toolset by ID", + "operationId": "fetch_mcp_toolset_v1_mcp_toolset__toolset_id__get", + "parameters": [ + { + "in": "path", + "name": "toolset_id", + "required": true, + "schema": { + "title": "Toolset Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fetch Mcp Toolset", + "tags": [ + "mcp_management" + ] + } + }, + "/v1/mcp/user-credentials": { + "get": { + "description": "List all OAuth2 MCP credentials stored for the calling user", + "operationId": "list_mcp_user_credentials_v1_mcp_user_credentials_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/MCPUserCredentialListItem" + }, + "title": "Response List Mcp User Credentials V1 Mcp User Credentials Get", + "type": "array" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Mcp User Credentials", + "tags": [ + "mcp_management" + ] + } + } + } + }, + "mcp_rest": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "MCPCredentials": { + "properties": { + "auth_value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Value" + }, + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Access Key Id" + }, + "aws_region_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Region Name" + }, + "aws_role_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Role Name" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Secret Access Key" + }, + "aws_service_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Service Name" + }, + "aws_session_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Name" + }, + "aws_session_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Aws Session Token" + }, + "client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Id" + }, + "client_secret": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Client Secret" + }, + "scopes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Scopes" + } + }, + "title": "MCPCredentials", + "type": "object" + }, + "NewMCPServerRequest": { + "properties": { + "alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Alias" + }, + "allow_all_keys": { + "default": false, + "title": "Allow All Keys", + "type": "boolean" + }, + "allowed_tools": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Allowed Tools" + }, + "approval_status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Approval Status" + }, + "args": { + "items": { + "type": "string" + }, + "title": "Args", + "type": "array" + }, + "auth_type": { + "anyOf": [ + { + "enum": [ + "none", + "api_key", + "bearer_token", + "basic", + "authorization", + "oauth2", + "aws_sigv4", + "token" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Auth Type" + }, + "authorization_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization Url" + }, + "available_on_public_internet": { + "default": true, + "title": "Available On Public Internet", + "type": "boolean" + }, + "byok_api_key_help_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Byok Api Key Help Url" + }, + "byok_description": { + "items": { + "type": "string" + }, + "title": "Byok Description", + "type": "array" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Command" + }, + "credentials": { + "anyOf": [ + { + "$ref": "#/components/schemas/MCPCredentials" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "extra_headers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Extra Headers" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "is_byok": { + "default": false, + "title": "Is Byok", + "type": "boolean" + }, + "mcp_access_groups": { + "items": { + "type": "string" + }, + "title": "Mcp Access Groups", + "type": "array" + }, + "mcp_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mcp Info" + }, + "oauth2_flow": { + "anyOf": [ + { + "enum": [ + "client_credentials", + "authorization_code" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Oauth2 Flow" + }, + "registration_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Registration Url" + }, + "server_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Id" + }, + "server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Server Name" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source Url" + }, + "spec_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Spec Path" + }, + "static_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Static Headers" + }, + "submitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted At" + }, + "submitted_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Server-managed: set by the endpoint; caller values are overridden.", + "title": "Submitted By" + }, + "token_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token Url" + }, + "tool_name_to_description": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Description" + }, + "tool_name_to_display_name": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Tool Name To Display Name" + }, + "transport": { + "default": "sse", + "enum": [ + "sse", + "http", + "stdio" + ], + "title": "Transport", + "type": "string" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + } + }, + "title": "NewMCPServerRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/mcp-rest/test/connection": { + "post": { + "description": "Test if we can connect to the provided MCP server before adding it", + "operationId": "test_connection_mcp_rest_test_connection_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Connection", + "tags": [ + "mcp_rest" + ] + } + }, + "/mcp-rest/test/tools/list": { + "post": { + "description": "Preview tools available from MCP server before adding it", + "operationId": "test_tools_list_mcp_rest_test_tools_list_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMCPServerRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Tools List", + "tags": [ + "mcp_rest" + ] + } + }, + "/mcp-rest/tools/call": { + "post": { + "description": "REST API to call a specific MCP tool with the provided arguments", + "operationId": "call_tool_rest_api_mcp_rest_tools_call_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Call Tool Rest Api", + "tags": [ + "mcp_rest" + ] + } + }, + "/mcp-rest/tools/list": { + "get": { + "description": "List all available tools with information about the server they belong to.\n\nExample response:\n{\n \"tools\": [\n {\n \"name\": \"create_zap\",\n \"description\": \"Create a new zap\",\n \"inputSchema\": \"tool_input_schema\",\n \"mcp_info\": {\n \"server_name\": \"zapier\",\n \"logo_url\": \"https://www.zapier.com/logo.png\",\n }\n }\n ],\n \"error\": null,\n \"message\": \"Successfully retrieved tools\"\n}", + "operationId": "list_tool_rest_api_mcp_rest_tools_list_get", + "parameters": [ + { + "description": "The server id to list tools for", + "in": "query", + "name": "server_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The server id to list tools for", + "title": "Server Id" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response List Tool Rest Api Mcp Rest Tools List Get", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Tool Rest Api", + "tags": [ + "mcp_rest" + ] + } + } + } + }, + "policies": { + "components": { + "schemas": { + "ChatCompletionAssistantMessage": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatCompletionTextObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionThinkingBlock" + }, + { + "$ref": "#/components/schemas/ChatCompletionRedactedThinkingBlock" + }, + { + "$ref": "#/components/schemas/ChatCompletionImageObject" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Content" + }, + "function_call": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatCompletionToolCallFunctionChunk" + }, + { + "type": "null" + } + ] + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "reasoning_content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reasoning Content" + }, + "reasoning_items": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ChatCompletionReasoningItem" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Reasoning Items" + }, + "role": { + "const": "assistant", + "title": "Role", + "type": "string" + }, + "thinking_blocks": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatCompletionThinkingBlock" + }, + { + "$ref": "#/components/schemas/ChatCompletionRedactedThinkingBlock" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Thinking Blocks" + }, + "tool_calls": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ChatCompletionAssistantToolCall" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tool Calls" + } + }, + "required": [ + "role" + ], + "title": "ChatCompletionAssistantMessage", + "type": "object" + }, + "ChatCompletionAssistantToolCall": { + "properties": { + "function": { + "$ref": "#/components/schemas/ChatCompletionToolCallFunctionChunk" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "type": { + "const": "function", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "type", + "function" + ], + "title": "ChatCompletionAssistantToolCall", + "type": "object" + }, + "ChatCompletionAudioObject": { + "properties": { + "input_audio": { + "$ref": "#/components/schemas/InputAudio" + }, + "type": { + "const": "input_audio", + "title": "Type", + "type": "string" + } + }, + "required": [ + "input_audio", + "type" + ], + "title": "ChatCompletionAudioObject", + "type": "object" + }, + "ChatCompletionCachedContent": { + "properties": { + "type": { + "const": "ephemeral", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "ChatCompletionCachedContent", + "type": "object" + }, + "ChatCompletionDeveloperMessage": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": {}, + "type": "array" + } + ], + "title": "Content" + }, + "name": { + "title": "Name", + "type": "string" + }, + "role": { + "const": "developer", + "title": "Role", + "type": "string" + } + }, + "required": [ + "role", + "content" + ], + "title": "ChatCompletionDeveloperMessage", + "type": "object" + }, + "ChatCompletionDocumentObject": { + "properties": { + "citations": { + "anyOf": [ + { + "$ref": "#/components/schemas/CitationsObject" + }, + { + "type": "null" + } + ] + }, + "context": { + "title": "Context", + "type": "string" + }, + "source": { + "$ref": "#/components/schemas/DocumentObject" + }, + "title": { + "title": "Title", + "type": "string" + }, + "type": { + "const": "document", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "source", + "title", + "context", + "citations" + ], + "title": "ChatCompletionDocumentObject", + "type": "object" + }, + "ChatCompletionFileObject": { + "properties": { + "file": { + "$ref": "#/components/schemas/ChatCompletionFileObjectFile" + }, + "type": { + "const": "file", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "file" + ], + "title": "ChatCompletionFileObject", + "type": "object" + }, + "ChatCompletionFileObjectFile": { + "properties": { + "detail": { + "title": "Detail", + "type": "string" + }, + "file_data": { + "title": "File Data", + "type": "string" + }, + "file_id": { + "title": "File Id", + "type": "string" + }, + "filename": { + "title": "Filename", + "type": "string" + }, + "format": { + "title": "Format", + "type": "string" + }, + "video_metadata": { + "additionalProperties": true, + "title": "Video Metadata", + "type": "object" + } + }, + "title": "ChatCompletionFileObjectFile", + "type": "object" + }, + "ChatCompletionFunctionMessage": { + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "$ref": "#/components/schemas/ChatCompletionTextObject" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Content" + }, + "name": { + "title": "Name", + "type": "string" + }, + "role": { + "const": "function", + "title": "Role", + "type": "string" + }, + "tool_call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tool Call Id" + } + }, + "required": [ + "role", + "content", + "name", + "tool_call_id" + ], + "title": "ChatCompletionFunctionMessage", + "type": "object" + }, + "ChatCompletionImageObject": { + "properties": { + "image_url": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/components/schemas/ChatCompletionImageUrlObject" + } + ], + "title": "Image Url" + }, + "type": { + "const": "image_url", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "ChatCompletionImageObject", + "type": "object" + }, + "ChatCompletionImageUrlObject": { + "properties": { + "detail": { + "title": "Detail", + "type": "string" + }, + "format": { + "title": "Format", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "ChatCompletionImageUrlObject", + "type": "object" + }, + "ChatCompletionMessageToolCall": { + "additionalProperties": true, + "properties": {}, + "title": "ChatCompletionMessageToolCall", + "type": "object" + }, + "ChatCompletionReasoningItem": { + "description": "Represents an OpenAI Responses API reasoning item for round-tripping in conversation history.", + "properties": { + "encrypted_content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Encrypted Content" + }, + "id": { + "title": "Id", + "type": "string" + }, + "summary": { + "items": { + "$ref": "#/components/schemas/ChatCompletionReasoningSummaryTextBlock" + }, + "title": "Summary", + "type": "array" + }, + "type": { + "const": "reasoning", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "ChatCompletionReasoningItem", + "type": "object" + }, + "ChatCompletionReasoningSummaryTextBlock": { + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "summary_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "ChatCompletionReasoningSummaryTextBlock", + "type": "object" + }, + "ChatCompletionRedactedThinkingBlock": { + "properties": { + "cache_control": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + { + "type": "null" + } + ], + "title": "Cache Control" + }, + "data": { + "title": "Data", + "type": "string" + }, + "type": { + "const": "redacted_thinking", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "ChatCompletionRedactedThinkingBlock", + "type": "object" + }, + "ChatCompletionSystemMessage": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": {}, + "type": "array" + } + ], + "title": "Content" + }, + "name": { + "title": "Name", + "type": "string" + }, + "role": { + "const": "system", + "title": "Role", + "type": "string" + } + }, + "required": [ + "role", + "content" + ], + "title": "ChatCompletionSystemMessage", + "type": "object" + }, + "ChatCompletionTextObject": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "ChatCompletionTextObject", + "type": "object" + }, + "ChatCompletionThinkingBlock": { + "properties": { + "cache_control": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + { + "type": "null" + } + ], + "title": "Cache Control" + }, + "signature": { + "title": "Signature", + "type": "string" + }, + "thinking": { + "title": "Thinking", + "type": "string" + }, + "type": { + "const": "thinking", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "ChatCompletionThinkingBlock", + "type": "object" + }, + "ChatCompletionToolCallChunk": { + "properties": { + "function": { + "$ref": "#/components/schemas/ChatCompletionToolCallFunctionChunk" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "index": { + "title": "Index", + "type": "integer" + }, + "type": { + "const": "function", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "type", + "function", + "index" + ], + "title": "ChatCompletionToolCallChunk", + "type": "object" + }, + "ChatCompletionToolCallFunctionChunk": { + "properties": { + "arguments": { + "title": "Arguments", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "provider_specific_fields": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Provider Specific Fields" + } + }, + "title": "ChatCompletionToolCallFunctionChunk", + "type": "object" + }, + "ChatCompletionToolMessage": { + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "$ref": "#/components/schemas/ChatCompletionTextObject" + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "tool", + "title": "Role", + "type": "string" + }, + "tool_call_id": { + "title": "Tool Call Id", + "type": "string" + } + }, + "required": [ + "role", + "content", + "tool_call_id" + ], + "title": "ChatCompletionToolMessage", + "type": "object" + }, + "ChatCompletionToolParam": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "function": { + "$ref": "#/components/schemas/ChatCompletionToolParamFunctionChunk" + }, + "type": { + "anyOf": [ + { + "const": "function", + "type": "string" + }, + { + "type": "string" + } + ], + "title": "Type" + } + }, + "required": [ + "type", + "function" + ], + "title": "ChatCompletionToolParam", + "type": "object" + }, + "ChatCompletionToolParamFunctionChunk": { + "properties": { + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "parameters": { + "additionalProperties": true, + "title": "Parameters", + "type": "object" + }, + "strict": { + "title": "Strict", + "type": "boolean" + } + }, + "required": [ + "name" + ], + "title": "ChatCompletionToolParamFunctionChunk", + "type": "object" + }, + "ChatCompletionUserMessage": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatCompletionTextObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionImageObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionAudioObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionDocumentObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionVideoObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionFileObject" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "title": "Role", + "type": "string" + } + }, + "required": [ + "role", + "content" + ], + "title": "ChatCompletionUserMessage", + "type": "object" + }, + "ChatCompletionVideoObject": { + "properties": { + "type": { + "const": "video_url", + "title": "Type", + "type": "string" + }, + "video_url": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/components/schemas/ChatCompletionVideoUrlObject" + } + ], + "title": "Video Url" + } + }, + "required": [ + "type", + "video_url" + ], + "title": "ChatCompletionVideoObject", + "type": "object" + }, + "ChatCompletionVideoUrlObject": { + "properties": { + "detail": { + "title": "Detail", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "ChatCompletionVideoUrlObject", + "type": "object" + }, + "CitationsObject": { + "properties": { + "enabled": { + "title": "Enabled", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "title": "CitationsObject", + "type": "object" + }, + "DocumentObject": { + "properties": { + "data": { + "title": "Data", + "type": "string" + }, + "media_type": { + "title": "Media Type", + "type": "string" + }, + "type": { + "const": "text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "media_type", + "data" + ], + "title": "DocumentObject", + "type": "object" + }, + "EnrichTemplateRequest": { + "properties": { + "competitors": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "maxItems": 100, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Optional list of competitor names", + "title": "Competitors" + }, + "instruction": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Refinement instruction for modifying the competitor list (e.g. 'add 10 more from Asia')", + "title": "Instruction" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "parameters": { + "additionalProperties": true, + "title": "Parameters", + "type": "object" + }, + "template_id": { + "title": "Template Id", + "type": "string" + } + }, + "required": [ + "template_id", + "parameters" + ], + "title": "EnrichTemplateRequest", + "type": "object" + }, + "GenericGuardrailAPIInputs": { + "properties": { + "images": { + "items": { + "type": "string" + }, + "title": "Images", + "type": "array" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "structured_messages": { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/ChatCompletionUserMessage" + }, + { + "$ref": "#/components/schemas/ChatCompletionAssistantMessage" + }, + { + "$ref": "#/components/schemas/ChatCompletionToolMessage" + }, + { + "$ref": "#/components/schemas/ChatCompletionSystemMessage" + }, + { + "$ref": "#/components/schemas/ChatCompletionFunctionMessage" + }, + { + "$ref": "#/components/schemas/ChatCompletionDeveloperMessage" + } + ] + }, + "title": "Structured Messages", + "type": "array" + }, + "texts": { + "items": { + "type": "string" + }, + "title": "Texts", + "type": "array" + }, + "tool_calls": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ChatCompletionToolCallChunk" + }, + "type": "array" + }, + { + "items": { + "$ref": "#/components/schemas/ChatCompletionMessageToolCall" + }, + "type": "array" + } + ], + "title": "Tool Calls" + }, + "tools": { + "items": { + "$ref": "#/components/schemas/ChatCompletionToolParam" + }, + "title": "Tools", + "type": "array" + } + }, + "title": "GenericGuardrailAPIInputs", + "type": "object" + }, + "GuardrailTestResultEntry": { + "properties": { + "action": { + "title": "Action", + "type": "string" + }, + "details": { + "title": "Details", + "type": "string" + }, + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + }, + "output_text": { + "title": "Output Text", + "type": "string" + } + }, + "required": [ + "guardrail_name", + "action", + "output_text", + "details" + ], + "title": "GuardrailTestResultEntry", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "InputAudio": { + "properties": { + "data": { + "title": "Data", + "type": "string" + }, + "format": { + "enum": [ + "wav", + "mp3" + ], + "title": "Format", + "type": "string" + } + }, + "required": [ + "data", + "format" + ], + "title": "InputAudio", + "type": "object" + }, + "PolicyGuardrailsResponse": { + "description": "Guardrails configuration for a policy.", + "properties": { + "add": { + "items": { + "type": "string" + }, + "title": "Add", + "type": "array" + }, + "remove": { + "items": { + "type": "string" + }, + "title": "Remove", + "type": "array" + } + }, + "title": "PolicyGuardrailsResponse", + "type": "object" + }, + "PolicyInfoResponse": { + "description": "Response for /policy/info/{policy_name} endpoint.", + "properties": { + "guardrails": { + "$ref": "#/components/schemas/PolicyGuardrailsResponse" + }, + "inherit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Inherit" + }, + "inheritance_chain": { + "items": { + "type": "string" + }, + "title": "Inheritance Chain", + "type": "array" + }, + "policy_name": { + "title": "Policy Name", + "type": "string" + }, + "resolved_guardrails": { + "items": { + "type": "string" + }, + "title": "Resolved Guardrails", + "type": "array" + }, + "scope": { + "$ref": "#/components/schemas/PolicyScopeResponse" + } + }, + "required": [ + "policy_name", + "scope", + "guardrails", + "resolved_guardrails", + "inheritance_chain" + ], + "title": "PolicyInfoResponse", + "type": "object" + }, + "PolicyListResponse": { + "description": "Response for /policy/list endpoint.", + "properties": { + "policies": { + "additionalProperties": { + "$ref": "#/components/schemas/PolicySummaryItem" + }, + "title": "Policies", + "type": "object" + }, + "total_count": { + "title": "Total Count", + "type": "integer" + } + }, + "required": [ + "policies", + "total_count" + ], + "title": "PolicyListResponse", + "type": "object" + }, + "PolicyMatchContext": { + "additionalProperties": false, + "description": "Context used to match a request against policies.\n\nContains the team alias, key alias, and model from the incoming request.", + "properties": { + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "API key alias from the request.", + "title": "Key Alias" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Model name from the request.", + "title": "Model" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Tags from key/team metadata.", + "title": "Tags" + }, + "team_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Team alias from the request.", + "title": "Team Alias" + } + }, + "title": "PolicyMatchContext", + "type": "object" + }, + "PolicyScopeResponse": { + "description": "Scope configuration for a policy.", + "properties": { + "keys": { + "items": { + "type": "string" + }, + "title": "Keys", + "type": "array" + }, + "models": { + "items": { + "type": "string" + }, + "title": "Models", + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "teams": { + "items": { + "type": "string" + }, + "title": "Teams", + "type": "array" + } + }, + "title": "PolicyScopeResponse", + "type": "object" + }, + "PolicySummaryItem": { + "description": "Summary of a single policy for list endpoint.", + "properties": { + "guardrails": { + "$ref": "#/components/schemas/PolicyGuardrailsResponse" + }, + "inherit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Inherit" + }, + "inheritance_chain": { + "items": { + "type": "string" + }, + "title": "Inheritance Chain", + "type": "array" + }, + "resolved_guardrails": { + "items": { + "type": "string" + }, + "title": "Resolved Guardrails", + "type": "array" + }, + "scope": { + "$ref": "#/components/schemas/PolicyScopeResponse" + } + }, + "required": [ + "scope", + "guardrails", + "resolved_guardrails", + "inheritance_chain" + ], + "title": "PolicySummaryItem", + "type": "object" + }, + "PolicyTestResponse": { + "description": "Response for /policy/test endpoint.", + "properties": { + "context": { + "$ref": "#/components/schemas/PolicyMatchContext" + }, + "matching_policies": { + "items": { + "type": "string" + }, + "title": "Matching Policies", + "type": "array" + }, + "message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Message" + }, + "resolved_guardrails": { + "items": { + "type": "string" + }, + "title": "Resolved Guardrails", + "type": "array" + } + }, + "required": [ + "context", + "matching_policies", + "resolved_guardrails" + ], + "title": "PolicyTestResponse", + "type": "object" + }, + "PolicyValidateRequest": { + "additionalProperties": false, + "description": "Request body for the /policy/validate endpoint.", + "properties": { + "policies": { + "additionalProperties": true, + "description": "Policy configuration to validate. Map of policy names to policy definitions.", + "title": "Policies", + "type": "object" + } + }, + "required": [ + "policies" + ], + "title": "PolicyValidateRequest", + "type": "object" + }, + "PolicyValidationError": { + "additionalProperties": false, + "description": "Represents a validation error or warning for a policy.", + "properties": { + "error_type": { + "$ref": "#/components/schemas/PolicyValidationErrorType", + "description": "Type of validation error." + }, + "field": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Specific field that caused the error (e.g., 'guardrails.add', 'scope.teams').", + "title": "Field" + }, + "message": { + "description": "Human-readable error message.", + "title": "Message", + "type": "string" + }, + "policy_name": { + "description": "Name of the policy with the issue.", + "title": "Policy Name", + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The invalid value that caused the error.", + "title": "Value" + } + }, + "required": [ + "policy_name", + "error_type", + "message" + ], + "title": "PolicyValidationError", + "type": "object" + }, + "PolicyValidationErrorType": { + "description": "Types of validation errors that can occur.", + "enum": [ + "invalid_guardrail", + "invalid_team", + "invalid_key", + "invalid_model", + "invalid_inheritance", + "circular_inheritance", + "invalid_scope", + "invalid_syntax" + ], + "title": "PolicyValidationErrorType", + "type": "string" + }, + "PolicyValidationResponse": { + "additionalProperties": false, + "description": "Response from policy validation.\n\n- `valid`: True if no blocking errors were found\n- `errors`: List of blocking errors (prevent policy from being applied)\n- `warnings`: List of non-blocking warnings (policy can still be applied)", + "properties": { + "errors": { + "description": "List of blocking validation errors.", + "items": { + "$ref": "#/components/schemas/PolicyValidationError" + }, + "title": "Errors", + "type": "array" + }, + "valid": { + "description": "True if the policy configuration is valid.", + "title": "Valid", + "type": "boolean" + }, + "warnings": { + "description": "List of non-blocking validation warnings.", + "items": { + "$ref": "#/components/schemas/PolicyValidationError" + }, + "title": "Warnings", + "type": "array" + } + }, + "required": [ + "valid" + ], + "title": "PolicyValidationResponse", + "type": "object" + }, + "SuggestTemplatesRequest": { + "properties": { + "attack_examples": { + "items": { + "type": "string" + }, + "title": "Attack Examples", + "type": "array" + }, + "description": { + "default": "", + "title": "Description", + "type": "string" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + } + }, + "title": "SuggestTemplatesRequest", + "type": "object" + }, + "TestPoliciesAndGuardrailsRequest": { + "description": "Request body for POST /utils/test_policies_and_guardrails.", + "properties": { + "agent_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When set, call chat completion with this model/agent for each input and include the response in the result.", + "title": "Agent Id" + }, + "guardrail_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Guardrail names to apply directly", + "title": "Guardrail Names" + }, + "input_type": { + "default": "request", + "description": "Whether inputs are request or response", + "enum": [ + "request", + "response" + ], + "title": "Input Type", + "type": "string" + }, + "inputs_list": { + "default": [], + "description": "List of GenericGuardrailAPIInputs; each item processed separately (for batch compliance testing).", + "items": { + "$ref": "#/components/schemas/GenericGuardrailAPIInputs" + }, + "title": "Inputs List", + "type": "array" + }, + "policy_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Policy names to resolve guardrails from", + "title": "Policy Names" + }, + "request_data": { + "additionalProperties": true, + "description": "Request context (model, user_id, etc.)", + "title": "Request Data", + "type": "object" + } + }, + "title": "TestPoliciesAndGuardrailsRequest", + "type": "object" + }, + "TestPolicyTemplateRequest": { + "properties": { + "guardrail_definitions": { + "description": "All guardrailDefinitions from the policy template", + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Guardrail Definitions", + "type": "array" + }, + "text": { + "description": "Test input text to run guardrails against", + "title": "Text", + "type": "string" + } + }, + "required": [ + "guardrail_definitions", + "text" + ], + "title": "TestPolicyTemplateRequest", + "type": "object" + }, + "TestPolicyTemplateResponse": { + "properties": { + "overall_action": { + "title": "Overall Action", + "type": "string" + }, + "results": { + "items": { + "$ref": "#/components/schemas/GuardrailTestResultEntry" + }, + "title": "Results", + "type": "array" + } + }, + "required": [ + "overall_action", + "results" + ], + "title": "TestPolicyTemplateResponse", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/policy/info/{policy_name}": { + "get": { + "description": "Get detailed information about a specific policy.\n\nReturns:\n- Policy configuration\n- Resolved guardrails (after inheritance)\n- Inheritance chain", + "operationId": "get_policy_info_policy_info__policy_name__get", + "parameters": [ + { + "in": "path", + "name": "policy_name", + "required": true, + "schema": { + "title": "Policy Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyInfoResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Policy Info", + "tags": [ + "policies" + ] + } + }, + "/policy/list": { + "get": { + "description": "List all loaded policies with their resolved guardrails.\n\nReturns information about each policy including:\n- Inheritance configuration\n- Scope (teams, keys, models)\n- Guardrails to add/remove\n- Resolved guardrails (after inheritance)\n- Inheritance chain", + "operationId": "list_policies_policy_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyListResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Policies", + "tags": [ + "policies" + ] + } + }, + "/policy/templates": { + "get": { + "description": "Get policy templates for the UI (pre-configured guardrail combinations).\n\nFetches from GitHub with automatic fallback to local backup on failure.\nSet LITELLM_LOCAL_POLICY_TEMPLATES=true to skip GitHub and use local backup only.", + "operationId": "get_policy_templates_policy_templates_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": {}, + "title": "Response Get Policy Templates Policy Templates Get", + "type": "array" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Policy Templates", + "tags": [ + "policies" + ] + } + }, + "/policy/templates/enrich": { + "post": { + "description": "Enrich a policy template with LLM-discovered data (e.g. competitor names).\n\nCalls an onboarded LLM to discover competitors for the given brand name,\nthen returns enriched guardrailDefinitions with the discovered data populated.", + "operationId": "enrich_policy_template_policy_templates_enrich_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnrichTemplateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Enrich Policy Template Policy Templates Enrich Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Enrich Policy Template", + "tags": [ + "policies" + ] + } + }, + "/policy/templates/enrich/stream": { + "post": { + "description": "Stream competitor names as SSE events as the LLM generates them.\n\nEvents:\n- data: {\"type\": \"competitor\", \"name\": \"...\"} \u2014 each competitor as discovered\n- data: {\"type\": \"done\", \"competitors\": [...], \"competitor_variations\": {...}, \"guardrailDefinitions\": [...]}", + "operationId": "enrich_policy_template_stream_policy_templates_enrich_stream_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnrichTemplateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Enrich Policy Template Stream", + "tags": [ + "policies" + ] + } + }, + "/policy/templates/suggest": { + "post": { + "description": "Use AI to suggest policy templates based on attack examples and descriptions.\n\nCalls an LLM with tool calling to match user requirements to available templates.", + "operationId": "suggest_policy_templates_policy_templates_suggest_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuggestTemplatesRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Suggest Policy Templates Policy Templates Suggest Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Suggest Policy Templates", + "tags": [ + "policies" + ] + } + }, + "/policy/templates/test": { + "post": { + "description": "Test a policy template's guardrails against a text input without creating them.\n\nInstantiates temporary guardrails from the template definitions, runs them\nagainst the provided text, and returns per-guardrail results so users can\nverify the template solves their problem before creating it.", + "operationId": "test_policy_template_policy_templates_test_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestPolicyTemplateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestPolicyTemplateResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Policy Template", + "tags": [ + "policies" + ] + } + }, + "/policy/test": { + "post": { + "description": "Test which policies would match a given request context.\n\nThis is useful for debugging and understanding policy behavior.\n\nRequest body:\n```json\n{\n \"team_alias\": \"healthcare-team\",\n \"key_alias\": \"my-api-key\",\n \"model\": \"gpt-4\"\n}\n```\n\nReturns:\n- matching_policies: List of policy names that match\n- resolved_guardrails: Final list of guardrails that would be applied", + "operationId": "test_policy_matching_policy_test_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyMatchContext" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyTestResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Policy Matching", + "tags": [ + "policies" + ] + } + }, + "/policy/validate": { + "post": { + "description": "Validate a policy configuration before applying it.\n\nChecks:\n- All referenced guardrails exist in the guardrail registry\n- All non-wildcard team aliases exist in the database\n- All non-wildcard key aliases exist in the database\n- Inheritance chains are valid (no cycles, parents exist)\n- Scope patterns are syntactically valid\n\nReturns:\n- valid: True if the policy configuration is valid (no blocking errors)\n- errors: List of blocking validation errors\n- warnings: List of non-blocking validation warnings\n\nExample request:\n```json\n{\n \"policies\": {\n \"global-baseline\": {\n \"guardrails\": {\n \"add\": [\"pii_blocker\", \"phi_blocker\"]\n },\n \"scope\": {\n \"teams\": [\"*\"],\n \"keys\": [\"*\"],\n \"models\": [\"*\"]\n }\n },\n \"healthcare-compliance\": {\n \"inherit\": \"global-baseline\",\n \"guardrails\": {\n \"add\": [\"hipaa_audit\"]\n },\n \"scope\": {\n \"teams\": [\"healthcare-team\"]\n }\n }\n }\n}\n```", + "operationId": "validate_policy_policy_validate_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyValidateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyValidationResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Validate Policy", + "tags": [ + "policies" + ] + } + }, + "/utils/test_policies_and_guardrails": { + "post": { + "description": "Apply policies and/or guardrails to inputs (for compliance UI testing).\n\nUse inputs_list for batch testing: each input is processed as a separate call so\nper-input block/allow and errors are returned.\n\nUse inputs for a single call (legacy).", + "operationId": "test_policies_and_guardrails_utils_test_policies_and_guardrails_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestPoliciesAndGuardrailsRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Policies And Guardrails", + "tags": [ + "policies" + ] + } + } + } + }, + "policy_engine": { + "components": { + "schemas": { + "AttachmentImpactResponse": { + "description": "Response for estimating the impact of a policy attachment.", + "properties": { + "affected_keys_count": { + "default": 0, + "description": "Number of keys that would be affected (named + unnamed).", + "title": "Affected Keys Count", + "type": "integer" + }, + "affected_teams_count": { + "default": 0, + "description": "Number of teams that would be affected (named + unnamed).", + "title": "Affected Teams Count", + "type": "integer" + }, + "sample_keys": { + "description": "Sample of affected key aliases (up to 10).", + "items": { + "type": "string" + }, + "title": "Sample Keys", + "type": "array" + }, + "sample_teams": { + "description": "Sample of affected team aliases (up to 10).", + "items": { + "type": "string" + }, + "title": "Sample Teams", + "type": "array" + }, + "unnamed_keys_count": { + "default": 0, + "description": "Number of affected keys without an alias.", + "title": "Unnamed Keys Count", + "type": "integer" + }, + "unnamed_teams_count": { + "default": 0, + "description": "Number of affected teams without an alias.", + "title": "Unnamed Teams Count", + "type": "integer" + } + }, + "title": "AttachmentImpactResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "PipelineTestRequest": { + "description": "Request body for testing a guardrail pipeline with sample messages.", + "properties": { + "pipeline": { + "additionalProperties": true, + "description": "Pipeline definition with 'mode' and 'steps'.", + "title": "Pipeline", + "type": "object" + }, + "test_messages": { + "description": "Test messages to run through the pipeline, e.g. [{'role': 'user', 'content': '...'}].", + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "title": "Test Messages", + "type": "array" + } + }, + "required": [ + "pipeline", + "test_messages" + ], + "title": "PipelineTestRequest", + "type": "object" + }, + "PolicyAttachmentCreateRequest": { + "description": "Request body for creating a policy attachment.", + "properties": { + "keys": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Key aliases or patterns this attachment applies to.", + "title": "Keys" + }, + "models": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Model names or patterns this attachment applies to.", + "title": "Models" + }, + "policy_name": { + "description": "Name of the policy to attach.", + "title": "Policy Name", + "type": "string" + }, + "scope": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Use '*' for global scope (applies to all requests).", + "title": "Scope" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Tag patterns this attachment applies to. Supports wildcards (e.g., health-*).", + "title": "Tags" + }, + "teams": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Team aliases or patterns this attachment applies to.", + "title": "Teams" + } + }, + "required": [ + "policy_name" + ], + "title": "PolicyAttachmentCreateRequest", + "type": "object" + }, + "PolicyAttachmentDBResponse": { + "description": "Response for a policy attachment from the database.", + "properties": { + "attachment_id": { + "description": "Unique ID of the attachment.", + "title": "Attachment Id", + "type": "string" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When the attachment was created.", + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Who created the attachment.", + "title": "Created By" + }, + "keys": { + "description": "Key patterns.", + "items": { + "type": "string" + }, + "title": "Keys", + "type": "array" + }, + "models": { + "description": "Model patterns.", + "items": { + "type": "string" + }, + "title": "Models", + "type": "array" + }, + "policy_name": { + "description": "Name of the attached policy.", + "title": "Policy Name", + "type": "string" + }, + "scope": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Scope of the attachment.", + "title": "Scope" + }, + "tags": { + "description": "Tag patterns.", + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "teams": { + "description": "Team patterns.", + "items": { + "type": "string" + }, + "title": "Teams", + "type": "array" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When the attachment was last updated.", + "title": "Updated At" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Who last updated the attachment.", + "title": "Updated By" + } + }, + "required": [ + "attachment_id", + "policy_name" + ], + "title": "PolicyAttachmentDBResponse", + "type": "object" + }, + "PolicyAttachmentListResponse": { + "description": "Response for listing policy attachments.", + "properties": { + "attachments": { + "description": "List of policy attachments.", + "items": { + "$ref": "#/components/schemas/PolicyAttachmentDBResponse" + }, + "title": "Attachments", + "type": "array" + }, + "total_count": { + "default": 0, + "description": "Total number of attachments.", + "title": "Total Count", + "type": "integer" + } + }, + "title": "PolicyAttachmentListResponse", + "type": "object" + }, + "PolicyConditionRequest": { + "description": "Condition for when a policy applies.", + "properties": { + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Model name pattern (exact match or regex) for when policy applies.", + "title": "Model" + } + }, + "title": "PolicyConditionRequest", + "type": "object" + }, + "PolicyCreateRequest": { + "description": "Request body for creating a new policy.", + "properties": { + "condition": { + "anyOf": [ + { + "$ref": "#/components/schemas/PolicyConditionRequest" + }, + { + "type": "null" + } + ], + "description": "Condition for when this policy applies." + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Human-readable description of the policy.", + "title": "Description" + }, + "guardrails_add": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of guardrail names to add.", + "title": "Guardrails Add" + }, + "guardrails_remove": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of guardrail names to remove (from inherited).", + "title": "Guardrails Remove" + }, + "inherit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of parent policy to inherit from.", + "title": "Inherit" + }, + "pipeline": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional guardrail pipeline for ordered execution. Contains 'mode' and 'steps'.", + "title": "Pipeline" + }, + "policy_name": { + "description": "Unique name for the policy.", + "title": "Policy Name", + "type": "string" + } + }, + "required": [ + "policy_name" + ], + "title": "PolicyCreateRequest", + "type": "object" + }, + "PolicyDBResponse": { + "description": "Response for a policy from the database.", + "properties": { + "condition": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Policy condition.", + "title": "Condition" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When the policy was created.", + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Who created the policy.", + "title": "Created By" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Policy description.", + "title": "Description" + }, + "guardrails_add": { + "description": "Guardrails to add.", + "items": { + "type": "string" + }, + "title": "Guardrails Add", + "type": "array" + }, + "guardrails_remove": { + "description": "Guardrails to remove.", + "items": { + "type": "string" + }, + "title": "Guardrails Remove", + "type": "array" + }, + "inherit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Parent policy name.", + "title": "Inherit" + }, + "is_latest": { + "default": true, + "description": "True if this is the latest version by version_number.", + "title": "Is Latest", + "type": "boolean" + }, + "parent_version_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Policy ID this version was cloned from.", + "title": "Parent Version Id" + }, + "pipeline": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional guardrail pipeline.", + "title": "Pipeline" + }, + "policy_id": { + "description": "Unique ID of the policy.", + "title": "Policy Id", + "type": "string" + }, + "policy_name": { + "description": "Name of the policy.", + "title": "Policy Name", + "type": "string" + }, + "production_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When this version was promoted to production.", + "title": "Production At" + }, + "published_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When this version was published.", + "title": "Published At" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "When the policy was last updated.", + "title": "Updated At" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Who last updated the policy.", + "title": "Updated By" + }, + "version_number": { + "default": 1, + "description": "Version number of this policy.", + "title": "Version Number", + "type": "integer" + }, + "version_status": { + "default": "production", + "description": "One of: draft, published, production.", + "title": "Version Status", + "type": "string" + } + }, + "required": [ + "policy_id", + "policy_name" + ], + "title": "PolicyDBResponse", + "type": "object" + }, + "PolicyListDBResponse": { + "description": "Response for listing policies from the database.", + "properties": { + "policies": { + "description": "List of policies.", + "items": { + "$ref": "#/components/schemas/PolicyDBResponse" + }, + "title": "Policies", + "type": "array" + }, + "total_count": { + "default": 0, + "description": "Total number of policies.", + "title": "Total Count", + "type": "integer" + } + }, + "title": "PolicyListDBResponse", + "type": "object" + }, + "PolicyMatchDetail": { + "description": "Details about why a specific policy matched.", + "properties": { + "guardrails_added": { + "description": "Guardrails this policy contributes.", + "items": { + "type": "string" + }, + "title": "Guardrails Added", + "type": "array" + }, + "matched_via": { + "description": "How the policy was matched (e.g., 'tag:healthcare', 'team:health-team', 'scope:*').", + "title": "Matched Via", + "type": "string" + }, + "policy_name": { + "description": "Name of the matched policy.", + "title": "Policy Name", + "type": "string" + } + }, + "required": [ + "policy_name", + "matched_via" + ], + "title": "PolicyMatchDetail", + "type": "object" + }, + "PolicyResolveRequest": { + "description": "Request body for resolving effective policies/guardrails for a context.", + "properties": { + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Key alias to resolve for.", + "title": "Key Alias" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Model name to resolve for.", + "title": "Model" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Tags to resolve for.", + "title": "Tags" + }, + "team_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Team alias to resolve for.", + "title": "Team Alias" + } + }, + "title": "PolicyResolveRequest", + "type": "object" + }, + "PolicyResolveResponse": { + "description": "Response for resolving effective policies/guardrails for a context.", + "properties": { + "effective_guardrails": { + "description": "Final list of guardrails that would be applied.", + "items": { + "type": "string" + }, + "title": "Effective Guardrails", + "type": "array" + }, + "matched_policies": { + "description": "Details about each matched policy and why it matched.", + "items": { + "$ref": "#/components/schemas/PolicyMatchDetail" + }, + "title": "Matched Policies", + "type": "array" + } + }, + "title": "PolicyResolveResponse", + "type": "object" + }, + "PolicyUpdateRequest": { + "description": "Request body for updating a policy.", + "properties": { + "condition": { + "anyOf": [ + { + "$ref": "#/components/schemas/PolicyConditionRequest" + }, + { + "type": "null" + } + ], + "description": "Condition for when this policy applies." + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Human-readable description of the policy.", + "title": "Description" + }, + "guardrails_add": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of guardrail names to add.", + "title": "Guardrails Add" + }, + "guardrails_remove": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of guardrail names to remove (from inherited).", + "title": "Guardrails Remove" + }, + "inherit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Name of parent policy to inherit from.", + "title": "Inherit" + }, + "pipeline": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional guardrail pipeline for ordered execution. Contains 'mode' and 'steps'.", + "title": "Pipeline" + }, + "policy_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New name for the policy.", + "title": "Policy Name" + } + }, + "title": "PolicyUpdateRequest", + "type": "object" + }, + "PolicyVersionCompareResponse": { + "description": "Response for comparing two policy versions.", + "properties": { + "field_diffs": { + "additionalProperties": { + "additionalProperties": true, + "type": "object" + }, + "description": "Field name -> {version_a: val, version_b: val} for differing fields.", + "title": "Field Diffs", + "type": "object" + }, + "version_a": { + "$ref": "#/components/schemas/PolicyDBResponse", + "description": "First version." + }, + "version_b": { + "$ref": "#/components/schemas/PolicyDBResponse", + "description": "Second version." + } + }, + "required": [ + "version_a", + "version_b" + ], + "title": "PolicyVersionCompareResponse", + "type": "object" + }, + "PolicyVersionCreateRequest": { + "description": "Request body for creating a new policy version (draft).", + "properties": { + "source_policy_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Policy ID to clone from. If None, clone from current production version.", + "title": "Source Policy Id" + } + }, + "title": "PolicyVersionCreateRequest", + "type": "object" + }, + "PolicyVersionListResponse": { + "description": "Response for listing all versions of a policy.", + "properties": { + "policy_name": { + "description": "Name of the policy.", + "title": "Policy Name", + "type": "string" + }, + "total_count": { + "default": 0, + "description": "Total number of versions.", + "title": "Total Count", + "type": "integer" + }, + "versions": { + "description": "All versions ordered by version_number desc.", + "items": { + "$ref": "#/components/schemas/PolicyDBResponse" + }, + "title": "Versions", + "type": "array" + } + }, + "required": [ + "policy_name" + ], + "title": "PolicyVersionListResponse", + "type": "object" + }, + "PolicyVersionStatusUpdateRequest": { + "description": "Request body for updating a policy version's status.", + "properties": { + "version_status": { + "description": "New status: 'published' or 'production'.", + "title": "Version Status", + "type": "string" + } + }, + "required": [ + "version_status" + ], + "title": "PolicyVersionStatusUpdateRequest", + "type": "object" + }, + "UsageOverviewResponse": { + "properties": { + "chart": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Chart", + "type": "array" + }, + "passRate": { + "title": "Passrate", + "type": "number" + }, + "rows": { + "items": { + "$ref": "#/components/schemas/UsageOverviewRow" + }, + "title": "Rows", + "type": "array" + }, + "totalBlocked": { + "title": "Totalblocked", + "type": "integer" + }, + "totalRequests": { + "title": "Totalrequests", + "type": "integer" + } + }, + "required": [ + "rows", + "chart", + "totalRequests", + "totalBlocked", + "passRate" + ], + "title": "UsageOverviewResponse", + "type": "object" + }, + "UsageOverviewRow": { + "properties": { + "avgLatency": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avglatency" + }, + "avgScore": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Avgscore" + }, + "failRate": { + "title": "Failrate", + "type": "number" + }, + "id": { + "title": "Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "requestsEvaluated": { + "title": "Requestsevaluated", + "type": "integer" + }, + "status": { + "title": "Status", + "type": "string" + }, + "trend": { + "title": "Trend", + "type": "string" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "name", + "type", + "provider", + "requestsEvaluated", + "failRate", + "avgScore", + "avgLatency", + "status", + "trend" + ], + "title": "UsageOverviewRow", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/policies": { + "post": { + "description": "Create a new policy.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"policy_name\": \"global-baseline\",\n \"description\": \"Base guardrails for all requests\",\n \"guardrails_add\": [\"pii_masking\", \"prompt_injection\"],\n \"guardrails_remove\": []\n }'\n```\n\nExample Response:\n```json\n{\n \"policy_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"policy_name\": \"global-baseline\",\n \"inherit\": null,\n \"description\": \"Base guardrails for all requests\",\n \"guardrails_add\": [\"pii_masking\", \"prompt_injection\"],\n \"guardrails_remove\": [],\n \"condition\": null,\n \"created_at\": \"2024-01-01T00:00:00Z\",\n \"updated_at\": \"2024-01-01T00:00:00Z\"\n}\n```", + "operationId": "create_policy_policies_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Policy", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/attachments": { + "post": { + "description": "Create a new policy attachment.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/attachments\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"policy_name\": \"global-baseline\",\n \"scope\": \"*\"\n }'\n```\n\nExample with team-specific attachment:\n```bash\ncurl -X POST \"http://localhost:4000/policies/attachments\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"policy_name\": \"healthcare-compliance\",\n \"teams\": [\"healthcare-team\", \"medical-research\"]\n }'\n```\n\nExample Response:\n```json\n{\n \"attachment_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"policy_name\": \"global-baseline\",\n \"scope\": \"*\",\n \"teams\": [],\n \"keys\": [],\n \"models\": [],\n \"created_at\": \"2024-01-01T00:00:00Z\",\n \"updated_at\": \"2024-01-01T00:00:00Z\"\n}\n```", + "operationId": "create_policy_attachment_policies_attachments_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Policy Attachment", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/attachments/estimate-impact": { + "post": { + "description": "Estimate how many keys and teams would be affected by a policy attachment.\n\nUse this before creating an attachment to preview the blast radius.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/attachments/estimate-impact\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"policy_name\": \"hipaa-compliance\",\n \"tags\": [\"healthcare\", \"health-*\"]\n }'\n```", + "operationId": "estimate_attachment_impact_policies_attachments_estimate_impact_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AttachmentImpactResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Estimate Attachment Impact", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/attachments/list": { + "get": { + "description": "List all policy attachments from the database.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/policies/attachments/list\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"attachments\": [\n {\n \"attachment_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"policy_name\": \"global-baseline\",\n \"scope\": \"*\",\n \"teams\": [],\n \"keys\": [],\n \"models\": [],\n \"created_at\": \"2024-01-01T00:00:00Z\",\n \"updated_at\": \"2024-01-01T00:00:00Z\"\n }\n ],\n \"total_count\": 1\n}\n```", + "operationId": "list_policy_attachments_policies_attachments_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentListResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Policy Attachments", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/attachments/{attachment_id}": { + "delete": { + "description": "Delete a policy attachment.\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/policies/attachments/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Attachment 123e4567-e89b-12d3-a456-426614174000 deleted successfully\"\n}\n```", + "operationId": "delete_policy_attachment_policies_attachments__attachment_id__delete", + "parameters": [ + { + "in": "path", + "name": "attachment_id", + "required": true, + "schema": { + "title": "Attachment Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Policy Attachment", + "tags": [ + "policy_engine" + ] + }, + "get": { + "description": "Get a policy attachment by ID.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/policies/attachments/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```", + "operationId": "get_policy_attachment_policies_attachments__attachment_id__get", + "parameters": [ + { + "in": "path", + "name": "attachment_id", + "required": true, + "schema": { + "title": "Attachment Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Policy Attachment", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/compare": { + "get": { + "description": "Compare two policy versions. Query params: version_a, version_b (policy version IDs).", + "operationId": "compare_policy_versions_policies_compare_get", + "parameters": [ + { + "in": "query", + "name": "version_a", + "required": true, + "schema": { + "title": "Version A", + "type": "string" + } + }, + { + "in": "query", + "name": "version_b", + "required": true, + "schema": { + "title": "Version B", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyVersionCompareResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Compare Policy Versions", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/list": { + "get": { + "description": "List all policies from the database. Optionally filter by version_status.\n\nQuery params:\n- version_status: Optional. One of \"draft\", \"published\", \"production\".\n If omitted, all versions are returned.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/policies/list\" \\\n -H \"Authorization: Bearer \"\ncurl -X GET \"http://localhost:4000/policies/list?version_status=production\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"policies\": [\n {\n \"policy_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"policy_name\": \"global-baseline\",\n \"version_number\": 1,\n \"version_status\": \"production\",\n \"inherit\": null,\n \"description\": \"Base guardrails for all requests\",\n \"guardrails_add\": [\"pii_masking\"],\n \"guardrails_remove\": [],\n \"condition\": null,\n \"created_at\": \"2024-01-01T00:00:00Z\",\n \"updated_at\": \"2024-01-01T00:00:00Z\"\n }\n ],\n \"total_count\": 1\n}\n```", + "operationId": "list_policies_policies_list_get", + "parameters": [ + { + "in": "query", + "name": "version_status", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version Status" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyListDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Policies", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/name/{policy_name}/all-versions": { + "delete": { + "description": "Delete all versions of a policy. Also removes from in-memory registry.", + "operationId": "delete_all_policy_versions_policies_name__policy_name__all_versions_delete", + "parameters": [ + { + "in": "path", + "name": "policy_name", + "required": true, + "schema": { + "title": "Policy Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete All Policy Versions", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/name/{policy_name}/versions": { + "get": { + "description": "List all versions of a policy by name, ordered by version_number descending.", + "operationId": "list_policy_versions_policies_name__policy_name__versions_get", + "parameters": [ + { + "in": "path", + "name": "policy_name", + "required": true, + "schema": { + "title": "Policy Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyVersionListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Policy Versions", + "tags": [ + "policy_engine" + ] + }, + "post": { + "description": "Create a new draft version of a policy. Copies all fields from the source.\nSource is current production if source_policy_id is not provided.", + "operationId": "create_policy_version_policies_name__policy_name__versions_post", + "parameters": [ + { + "in": "path", + "name": "policy_name", + "required": true, + "schema": { + "title": "Policy Name", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyVersionCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Policy Version", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/resolve": { + "post": { + "description": "Resolve which policies and guardrails apply for a given context.\n\nUse this endpoint to debug \"what guardrails would apply to a request\nwith this team/key/model/tags combination?\"\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/resolve\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"tags\": [\"healthcare\"],\n \"model\": \"gpt-4\"\n }'\n```", + "operationId": "resolve_policies_for_context_policies_resolve_post", + "parameters": [ + { + "description": "Force a DB sync before resolving. Default uses in-memory cache.", + "in": "query", + "name": "force_sync", + "required": false, + "schema": { + "default": false, + "description": "Force a DB sync before resolving. Default uses in-memory cache.", + "title": "Force Sync", + "type": "boolean" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyResolveRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyResolveResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Resolve Policies For Context", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/test-pipeline": { + "post": { + "description": "Test a guardrail pipeline with sample messages.\n\nExecutes the pipeline steps against the provided test messages and returns\nstep-by-step results showing which guardrails passed/failed, actions taken,\nand timing information.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/test-pipeline\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"pipeline\": {\n \"mode\": \"pre_call\",\n \"steps\": [\n {\"guardrail\": \"pii-guard\", \"on_pass\": \"next\", \"on_fail\": \"block\"}\n ]\n },\n \"test_messages\": [{\"role\": \"user\", \"content\": \"My SSN is 123-45-6789\"}]\n }'\n```", + "operationId": "test_pipeline_policies_test_pipeline_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PipelineTestRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Pipeline", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/usage/overview": { + "get": { + "description": "Return policy performance overview for the dashboard.", + "operationId": "policies_usage_overview_policies_usage_overview_get", + "parameters": [ + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "Start Date" + } + }, + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageOverviewResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Policies Usage Overview", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/{policy_id}": { + "delete": { + "description": "Delete a policy.\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/policies/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Policy 123e4567-e89b-12d3-a456-426614174000 deleted successfully\"\n}\n```", + "operationId": "delete_policy_policies__policy_id__delete", + "parameters": [ + { + "in": "path", + "name": "policy_id", + "required": true, + "schema": { + "title": "Policy Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Policy", + "tags": [ + "policy_engine" + ] + }, + "get": { + "description": "Get a policy by ID.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/policies/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```", + "operationId": "get_policy_policies__policy_id__get", + "parameters": [ + { + "in": "path", + "name": "policy_id", + "required": true, + "schema": { + "title": "Policy Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Policy", + "tags": [ + "policy_engine" + ] + }, + "put": { + "description": "Update an existing policy.\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/policies/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"description\": \"Updated description\",\n \"guardrails_add\": [\"pii_masking\", \"toxicity_filter\"]\n }'\n```", + "operationId": "update_policy_policies__policy_id__put", + "parameters": [ + { + "in": "path", + "name": "policy_id", + "required": true, + "schema": { + "title": "Policy Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Policy", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/{policy_id}/resolved-guardrails": { + "get": { + "description": "Get the resolved guardrails for a policy (including inherited guardrails).\n\nThis endpoint resolves the full inheritance chain and returns the final\nset of guardrails that would be applied for this policy.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/policies/123e4567-e89b-12d3-a456-426614174000/resolved-guardrails\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"policy_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"policy_name\": \"healthcare-compliance\",\n \"resolved_guardrails\": [\"pii_masking\", \"prompt_injection\", \"toxicity_filter\"]\n}\n```", + "operationId": "get_resolved_guardrails_policies__policy_id__resolved_guardrails_get", + "parameters": [ + { + "in": "path", + "name": "policy_id", + "required": true, + "schema": { + "title": "Policy Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Resolved Guardrails", + "tags": [ + "policy_engine" + ] + } + }, + "/policies/{policy_id}/status": { + "put": { + "description": "Update a policy version's status. Valid transitions:\n- draft -> published\n- published -> production (demotes current production to published)\n- production -> published (demotes, policy becomes inactive)", + "operationId": "update_policy_version_status_policies__policy_id__status_put", + "parameters": [ + { + "in": "path", + "name": "policy_id", + "required": true, + "schema": { + "title": "Policy Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyVersionStatusUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyDBResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Policy Version Status", + "tags": [ + "policy_engine" + ] + } + } + } + }, + "policy_resolve": { + "components": { + "schemas": { + "AttachmentImpactResponse": { + "description": "Response for estimating the impact of a policy attachment.", + "properties": { + "affected_keys_count": { + "default": 0, + "description": "Number of keys that would be affected (named + unnamed).", + "title": "Affected Keys Count", + "type": "integer" + }, + "affected_teams_count": { + "default": 0, + "description": "Number of teams that would be affected (named + unnamed).", + "title": "Affected Teams Count", + "type": "integer" + }, + "sample_keys": { + "description": "Sample of affected key aliases (up to 10).", + "items": { + "type": "string" + }, + "title": "Sample Keys", + "type": "array" + }, + "sample_teams": { + "description": "Sample of affected team aliases (up to 10).", + "items": { + "type": "string" + }, + "title": "Sample Teams", + "type": "array" + }, + "unnamed_keys_count": { + "default": 0, + "description": "Number of affected keys without an alias.", + "title": "Unnamed Keys Count", + "type": "integer" + }, + "unnamed_teams_count": { + "default": 0, + "description": "Number of affected teams without an alias.", + "title": "Unnamed Teams Count", + "type": "integer" + } + }, + "title": "AttachmentImpactResponse", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "PolicyAttachmentCreateRequest": { + "description": "Request body for creating a policy attachment.", + "properties": { + "keys": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Key aliases or patterns this attachment applies to.", + "title": "Keys" + }, + "models": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Model names or patterns this attachment applies to.", + "title": "Models" + }, + "policy_name": { + "description": "Name of the policy to attach.", + "title": "Policy Name", + "type": "string" + }, + "scope": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Use '*' for global scope (applies to all requests).", + "title": "Scope" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Tag patterns this attachment applies to. Supports wildcards (e.g., health-*).", + "title": "Tags" + }, + "teams": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Team aliases or patterns this attachment applies to.", + "title": "Teams" + } + }, + "required": [ + "policy_name" + ], + "title": "PolicyAttachmentCreateRequest", + "type": "object" + }, + "PolicyMatchDetail": { + "description": "Details about why a specific policy matched.", + "properties": { + "guardrails_added": { + "description": "Guardrails this policy contributes.", + "items": { + "type": "string" + }, + "title": "Guardrails Added", + "type": "array" + }, + "matched_via": { + "description": "How the policy was matched (e.g., 'tag:healthcare', 'team:health-team', 'scope:*').", + "title": "Matched Via", + "type": "string" + }, + "policy_name": { + "description": "Name of the matched policy.", + "title": "Policy Name", + "type": "string" + } + }, + "required": [ + "policy_name", + "matched_via" + ], + "title": "PolicyMatchDetail", + "type": "object" + }, + "PolicyResolveRequest": { + "description": "Request body for resolving effective policies/guardrails for a context.", + "properties": { + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Key alias to resolve for.", + "title": "Key Alias" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Model name to resolve for.", + "title": "Model" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Tags to resolve for.", + "title": "Tags" + }, + "team_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Team alias to resolve for.", + "title": "Team Alias" + } + }, + "title": "PolicyResolveRequest", + "type": "object" + }, + "PolicyResolveResponse": { + "description": "Response for resolving effective policies/guardrails for a context.", + "properties": { + "effective_guardrails": { + "description": "Final list of guardrails that would be applied.", + "items": { + "type": "string" + }, + "title": "Effective Guardrails", + "type": "array" + }, + "matched_policies": { + "description": "Details about each matched policy and why it matched.", + "items": { + "$ref": "#/components/schemas/PolicyMatchDetail" + }, + "title": "Matched Policies", + "type": "array" + } + }, + "title": "PolicyResolveResponse", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/policies/attachments/estimate-impact": { + "post": { + "description": "Estimate how many keys and teams would be affected by a policy attachment.\n\nUse this before creating an attachment to preview the blast radius.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/attachments/estimate-impact\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"policy_name\": \"hipaa-compliance\",\n \"tags\": [\"healthcare\", \"health-*\"]\n }'\n```", + "operationId": "estimate_attachment_impact_policies_attachments_estimate_impact_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyAttachmentCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AttachmentImpactResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Estimate Attachment Impact", + "tags": [ + "policy_resolve" + ] + } + }, + "/policies/resolve": { + "post": { + "description": "Resolve which policies and guardrails apply for a given context.\n\nUse this endpoint to debug \"what guardrails would apply to a request\nwith this team/key/model/tags combination?\"\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/policies/resolve\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"tags\": [\"healthcare\"],\n \"model\": \"gpt-4\"\n }'\n```", + "operationId": "resolve_policies_for_context_policies_resolve_post", + "parameters": [ + { + "description": "Force a DB sync before resolving. Default uses in-memory cache.", + "in": "query", + "name": "force_sync", + "required": false, + "schema": { + "default": false, + "description": "Force a DB sync before resolving. Default uses in-memory cache.", + "title": "Force Sync", + "type": "boolean" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyResolveRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PolicyResolveResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Resolve Policies For Context", + "tags": [ + "policy_resolve" + ] + } + } + } + }, + "prompts": { + "components": { + "schemas": { + "Body_convert_prompt_file_to_json_utils_dotprompt_json_converter_post": { + "properties": { + "file": { + "format": "binary", + "title": "File", + "type": "string" + } + }, + "required": [ + "file" + ], + "title": "Body_convert_prompt_file_to_json_utils_dotprompt_json_converter_post", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListPromptsResponse": { + "properties": { + "prompts": { + "items": { + "$ref": "#/components/schemas/PromptSpec" + }, + "title": "Prompts", + "type": "array" + } + }, + "required": [ + "prompts" + ], + "title": "ListPromptsResponse", + "type": "object" + }, + "PatchPromptRequest": { + "properties": { + "litellm_params": { + "anyOf": [ + { + "$ref": "#/components/schemas/PromptLiteLLMParams" + }, + { + "type": "null" + } + ] + }, + "prompt_info": { + "anyOf": [ + { + "$ref": "#/components/schemas/PromptInfo" + }, + { + "type": "null" + } + ] + } + }, + "title": "PatchPromptRequest", + "type": "object" + }, + "Prompt": { + "properties": { + "litellm_params": { + "$ref": "#/components/schemas/PromptLiteLLMParams" + }, + "prompt_id": { + "title": "Prompt Id", + "type": "string" + }, + "prompt_info": { + "anyOf": [ + { + "$ref": "#/components/schemas/PromptInfo" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "prompt_id", + "litellm_params" + ], + "title": "Prompt", + "type": "object" + }, + "PromptInfo": { + "additionalProperties": true, + "properties": { + "environment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "development", + "title": "Environment" + }, + "prompt_type": { + "enum": [ + "config", + "db" + ], + "title": "Prompt Type", + "type": "string" + } + }, + "required": [ + "prompt_type" + ], + "title": "PromptInfo", + "type": "object" + }, + "PromptInfoResponse": { + "properties": { + "environments": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Environments" + }, + "prompt_spec": { + "$ref": "#/components/schemas/PromptSpec" + }, + "raw_prompt_template": { + "anyOf": [ + { + "$ref": "#/components/schemas/PromptTemplateBase" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "prompt_spec" + ], + "title": "PromptInfoResponse", + "type": "object" + }, + "PromptLiteLLMParams": { + "additionalProperties": true, + "properties": { + "api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Base" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Key" + }, + "dotprompt_content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Dotprompt Content" + }, + "ignore_prompt_manager_model": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "title": "Ignore Prompt Manager Model" + }, + "ignore_prompt_manager_optional_params": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "title": "Ignore Prompt Manager Optional Params" + }, + "prompt_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Prompt Id" + }, + "prompt_integration": { + "title": "Prompt Integration", + "type": "string" + }, + "provider_specific_query_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Provider Specific Query Params" + } + }, + "required": [ + "prompt_integration" + ], + "title": "PromptLiteLLMParams", + "type": "object" + }, + "PromptSpec": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "environment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "development", + "title": "Environment" + }, + "litellm_params": { + "$ref": "#/components/schemas/PromptLiteLLMParams" + }, + "prompt_id": { + "title": "Prompt Id", + "type": "string" + }, + "prompt_info": { + "$ref": "#/components/schemas/PromptInfo" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "version": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "prompt_id", + "litellm_params", + "prompt_info" + ], + "title": "PromptSpec", + "type": "object" + }, + "PromptTemplateBase": { + "properties": { + "content": { + "title": "Content", + "type": "string" + }, + "litellm_prompt_id": { + "title": "Litellm Prompt Id", + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Metadata" + } + }, + "required": [ + "litellm_prompt_id", + "content" + ], + "title": "PromptTemplateBase", + "type": "object" + }, + "TestPromptRequest": { + "properties": { + "conversation_history": { + "anyOf": [ + { + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Conversation History" + }, + "dotprompt_content": { + "title": "Dotprompt Content", + "type": "string" + }, + "prompt_variables": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Prompt Variables" + } + }, + "required": [ + "dotprompt_content" + ], + "title": "TestPromptRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/prompts": { + "post": { + "description": "Create a new prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"json_prompt\",\n \"prompt_integration\": \"dotprompt\",\n ### EITHER prompt_directory OR prompt_data MUST BE PROVIDED\n \"prompt_directory\": \"/path/to/dotprompt/folder\",\n \"prompt_data\": {\"json_prompt\": {\"content\": \"This is a prompt\", \"metadata\": {\"model\": \"gpt-4\"}}}\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }'\n```", + "operationId": "create_prompt_prompts_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Prompt" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Prompt", + "tags": [ + "prompts" + ] + } + }, + "/prompts/list": { + "get": { + "description": "List the prompts that are available on the proxy server\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/prompts/list\" -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"prompts\": [\n {\n \"prompt_id\": \"my_prompt_id\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt_id\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_directory\": \"/path/to/prompts\"\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n }\n ]\n}\n```", + "operationId": "list_prompts_prompts_list_get", + "parameters": [ + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListPromptsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Prompts", + "tags": [ + "prompts" + ] + } + }, + "/prompts/test": { + "post": { + "description": "Test a prompt by rendering it with variables and executing an LLM call.\n\nThis endpoint allows testing prompts before saving them to the database.\nThe response is always streamed.\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts/test\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"dotprompt_content\": \"---\\nmodel: gpt-4o\\ntemperature: 0.7\\n---\\n\\nUser: Hello {{name}}\",\n \"prompt_variables\": {\n \"name\": \"World\"\n }\n }'\n```", + "operationId": "test_prompt_prompts_test_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestPromptRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Prompt", + "tags": [ + "prompts" + ] + } + }, + "/prompts/{prompt_id}": { + "delete": { + "description": "Delete a prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/prompts/my_prompt_id\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Prompt my_prompt_id deleted successfully\"\n}\n```", + "operationId": "delete_prompt_prompts__prompt_id__delete", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + }, + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Prompt", + "tags": [ + "prompts" + ] + }, + "get": { + "description": "Get detailed information about a specific prompt by ID, including prompt content\n\n \ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\n Example Request:\n ```bash\n curl -X GET \"http://localhost:4000/prompts/my_prompt_id/info\" \\\n -H \"Authorization: Bearer \"\n ```\n\n Example Response:\n ```json\n {\n \"prompt_id\": \"my_prompt_id\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt_id\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_directory\": \"/path/to/prompts\"\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\",\n \"content\": \"System: You are a helpful assistant.\n\nUser: {{user_message}}\"\n }\n ```", + "operationId": "get_prompt_info_prompts__prompt_id__get", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + }, + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PromptInfoResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Prompt Info", + "tags": [ + "prompts" + ] + }, + "patch": { + "description": "Partially update an existing prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nThis endpoint allows updating specific fields of a prompt without sending the entire object.\nOnly the following fields can be updated:\n- litellm_params: LiteLLM parameters for the prompt\n- prompt_info: Additional information about the prompt\n\nExample Request:\n```bash\ncurl -X PATCH \"http://localhost:4000/prompts/my_prompt_id\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_info\": {\n \"prompt_type\": \"db\"\n }\n }'\n```", + "operationId": "patch_prompt_prompts__prompt_id__patch", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + }, + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PatchPromptRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Patch Prompt", + "tags": [ + "prompts" + ] + }, + "put": { + "description": "Update an existing prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/prompts/my_prompt_id\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_directory\": \"/path/to/prompts\"\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }\n }'\n```", + "operationId": "update_prompt_prompts__prompt_id__put", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Prompt" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Prompt", + "tags": [ + "prompts" + ] + } + }, + "/prompts/{prompt_id}/info": { + "get": { + "description": "Get detailed information about a specific prompt by ID, including prompt content\n\n \ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\n Example Request:\n ```bash\n curl -X GET \"http://localhost:4000/prompts/my_prompt_id/info\" \\\n -H \"Authorization: Bearer \"\n ```\n\n Example Response:\n ```json\n {\n \"prompt_id\": \"my_prompt_id\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt_id\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_directory\": \"/path/to/prompts\"\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\",\n \"content\": \"System: You are a helpful assistant.\n\nUser: {{user_message}}\"\n }\n ```", + "operationId": "get_prompt_info_prompts__prompt_id__info_get", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + }, + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PromptInfoResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Prompt Info", + "tags": [ + "prompts" + ] + } + }, + "/prompts/{prompt_id}/versions": { + "get": { + "description": "Get all versions of a specific prompt by base prompt ID\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/prompts/jack_success/versions\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"prompts\": [\n {\n \"prompt_id\": \"jack_success.v1\",\n \"litellm_params\": {...},\n \"prompt_info\": {\"prompt_type\": \"db\"},\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n },\n {\n \"prompt_id\": \"jack_success.v2\",\n \"litellm_params\": {...},\n \"prompt_info\": {\"prompt_type\": \"db\"},\n \"created_at\": \"2023-11-09T13:45:12.345Z\",\n \"updated_at\": \"2023-11-09T13:45:12.345Z\"\n }\n ]\n}\n```", + "operationId": "get_prompt_versions_prompts__prompt_id__versions_get", + "parameters": [ + { + "in": "path", + "name": "prompt_id", + "required": true, + "schema": { + "title": "Prompt Id", + "type": "string" + } + }, + { + "in": "query", + "name": "environment", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Environment" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListPromptsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Prompt Versions", + "tags": [ + "prompts" + ] + } + }, + "/utils/dotprompt_json_converter": { + "post": { + "description": "Convert a .prompt file to JSON format.\n\nThis endpoint accepts a .prompt file upload and returns the equivalent JSON representation\nthat can be stored in a database or used programmatically.\n\nReturns the JSON structure with 'content' and 'metadata' fields.", + "operationId": "convert_prompt_file_to_json_utils_dotprompt_json_converter_post", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_convert_prompt_file_to_json_utils_dotprompt_json_converter_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Convert Prompt File To Json Utils Dotprompt Json Converter Post", + "type": "object" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Convert Prompt File To Json", + "tags": [ + "prompts" + ] + } + } + } + }, + "realtime": { + "components": { + "schemas": { + "RealtimeClientSecretResponse": { + "description": "Response from POST /v1/realtime/client_secrets.\n\nBoth the top-level `value` and `session.client_secret.value`\nwill contain the encrypted token instead of the raw ephemeral key.\nThe `session` field is kept as a raw dict so unknown fields pass through.", + "properties": { + "expires_at": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Expires At" + }, + "session": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Session" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "value" + ], + "title": "RealtimeClientSecretResponse", + "type": "object" + } + } + }, + "paths": { + "/openai/v1/realtime/calls": { + "post": { + "operationId": "proxy_realtime_calls_openai_v1_realtime_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/openai/v1/realtime/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_openai_v1_realtime_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, + "/realtime/calls": { + "post": { + "operationId": "proxy_realtime_calls_realtime_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/realtime/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_realtime_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, + "/v1/realtime/calls": { + "post": { + "operationId": "proxy_realtime_calls_v1_realtime_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/v1/realtime/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_v1_realtime_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + } + } + }, + "scim": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "SCIMFeature": { + "properties": { + "maxOperations": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Maxoperations" + }, + "maxPayloadSize": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Maxpayloadsize" + }, + "maxResults": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Maxresults" + }, + "supported": { + "title": "Supported", + "type": "boolean" + } + }, + "required": [ + "supported" + ], + "title": "SCIMFeature", + "type": "object" + }, + "SCIMGroup": { + "properties": { + "displayName": { + "title": "Displayname", + "type": "string" + }, + "externalId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Externalid" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "members": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/SCIMMember" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Members" + }, + "meta": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Meta" + }, + "schemas": { + "items": { + "type": "string" + }, + "title": "Schemas", + "type": "array" + } + }, + "required": [ + "schemas", + "displayName" + ], + "title": "SCIMGroup", + "type": "object" + }, + "SCIMListResponse": { + "properties": { + "Resources": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/SCIMUser" + }, + "type": "array" + }, + { + "items": { + "$ref": "#/components/schemas/SCIMGroup" + }, + "type": "array" + } + ], + "title": "Resources" + }, + "itemsPerPage": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 10, + "title": "Itemsperpage" + }, + "schemas": { + "default": [ + "urn:ietf:params:scim:api:messages:2.0:ListResponse" + ], + "items": { + "type": "string" + }, + "title": "Schemas", + "type": "array" + }, + "startIndex": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 1, + "title": "Startindex" + }, + "totalResults": { + "title": "Totalresults", + "type": "integer" + } + }, + "required": [ + "totalResults", + "Resources" + ], + "title": "SCIMListResponse", + "type": "object" + }, + "SCIMMember": { + "properties": { + "display": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Display" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "value" + ], + "title": "SCIMMember", + "type": "object" + }, + "SCIMPatchOp": { + "properties": { + "Operations": { + "items": { + "$ref": "#/components/schemas/SCIMPatchOperation" + }, + "title": "Operations", + "type": "array" + }, + "schemas": { + "default": [ + "urn:ietf:params:scim:api:messages:2.0:PatchOp" + ], + "items": { + "type": "string" + }, + "title": "Schemas", + "type": "array" + } + }, + "required": [ + "Operations" + ], + "title": "SCIMPatchOp", + "type": "object" + }, + "SCIMPatchOperation": { + "properties": { + "op": { + "title": "Op", + "type": "string" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Path" + }, + "value": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Value" + } + }, + "required": [ + "op" + ], + "title": "SCIMPatchOperation", + "type": "object" + }, + "SCIMServiceProviderConfig": { + "properties": { + "authenticationSchemes": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Authenticationschemes" + }, + "bulk": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": false + } + }, + "changePassword": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": false + } + }, + "etag": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": false + } + }, + "filter": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": false + } + }, + "meta": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Meta" + }, + "patch": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": true + } + }, + "schemas": { + "default": [ + "urn:ietf:params:scim:schemas:core:2.0:ServiceProviderConfig" + ], + "items": { + "type": "string" + }, + "title": "Schemas", + "type": "array" + }, + "sort": { + "$ref": "#/components/schemas/SCIMFeature", + "default": { + "supported": false + } + } + }, + "title": "SCIMServiceProviderConfig", + "type": "object" + }, + "SCIMUser": { + "properties": { + "active": { + "default": true, + "title": "Active", + "type": "boolean" + }, + "displayName": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Displayname" + }, + "emails": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/SCIMUserEmail" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Emails" + }, + "externalId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Externalid" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/SCIMUserGroup" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Groups" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "meta": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Meta" + }, + "name": { + "anyOf": [ + { + "$ref": "#/components/schemas/SCIMUserName" + }, + { + "type": "null" + } + ] + }, + "schemas": { + "items": { + "type": "string" + }, + "title": "Schemas", + "type": "array" + }, + "userName": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Username" + } + }, + "required": [ + "schemas" + ], + "title": "SCIMUser", + "type": "object" + }, + "SCIMUserEmail": { + "properties": { + "primary": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Primary" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + }, + "value": { + "format": "email", + "title": "Value", + "type": "string" + } + }, + "required": [ + "value" + ], + "title": "SCIMUserEmail", + "type": "object" + }, + "SCIMUserGroup": { + "properties": { + "display": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Display" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "direct", + "title": "Type" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "value" + ], + "title": "SCIMUserGroup", + "type": "object" + }, + "SCIMUserName": { + "properties": { + "familyName": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Familyname" + }, + "formatted": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Formatted" + }, + "givenName": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Givenname" + }, + "honorificPrefix": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Honorificprefix" + }, + "honorificSuffix": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Honorificsuffix" + }, + "middleName": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Middlename" + } + }, + "title": "SCIMUserName", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/scim/v2": { + "get": { + "description": "Base SCIM v2 endpoint for resource discovery per RFC 7644 Section 4.\n\nReturns a ListResponse of ResourceTypes supported by this SCIM service provider.\nIdentity providers (Okta, Azure AD, etc.) use this endpoint for resource discovery.", + "operationId": "get_scim_base_scim_v2_get", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Scim Base", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Groups": { + "get": { + "description": "Get a list of groups according to SCIM v2 protocol", + "operationId": "get_groups_scim_v2_Groups_get", + "parameters": [ + { + "in": "query", + "name": "startIndex", + "required": false, + "schema": { + "default": 1, + "minimum": 1, + "title": "Startindex", + "type": "integer" + } + }, + { + "in": "query", + "name": "count", + "required": false, + "schema": { + "default": 10, + "maximum": 100, + "minimum": 1, + "title": "Count", + "type": "integer" + } + }, + { + "in": "query", + "name": "filter", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Filter" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Groups", + "tags": [ + "scim" + ] + }, + "post": { + "description": "Create a group according to SCIM v2 protocol", + "operationId": "create_group_scim_v2_Groups_post", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Group", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Groups/{group_id}": { + "delete": { + "description": "Delete a group according to SCIM v2 protocol", + "operationId": "delete_group_scim_v2_Groups__group_id__delete", + "parameters": [ + { + "in": "path", + "name": "group_id", + "required": true, + "schema": { + "title": "Group ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Group", + "tags": [ + "scim" + ] + }, + "get": { + "description": "Get a single group by ID according to SCIM v2 protocol", + "operationId": "get_group_scim_v2_Groups__group_id__get", + "parameters": [ + { + "in": "path", + "name": "group_id", + "required": true, + "schema": { + "title": "Group ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Group", + "tags": [ + "scim" + ] + }, + "patch": { + "description": "Patch a group according to SCIM v2 protocol", + "operationId": "patch_group_scim_v2_Groups__group_id__patch", + "parameters": [ + { + "in": "path", + "name": "group_id", + "required": true, + "schema": { + "title": "Group ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMPatchOp" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Patch Group", + "tags": [ + "scim" + ] + }, + "put": { + "description": "Update a group according to SCIM v2 protocol", + "operationId": "update_group_scim_v2_Groups__group_id__put", + "parameters": [ + { + "in": "path", + "name": "group_id", + "required": true, + "schema": { + "title": "Group ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMGroup" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Group", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/ResourceTypes": { + "get": { + "description": "SCIM ResourceTypes endpoint per RFC 7644 Section 4.\n\nReturns a ListResponse of all resource types supported by this service provider.", + "operationId": "get_resource_types_scim_v2_ResourceTypes_get", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Resource Types", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/ResourceTypes/{resource_type_id}": { + "get": { + "description": "Get a single ResourceType by ID per RFC 7644.", + "operationId": "get_resource_type_scim_v2_ResourceTypes__resource_type_id__get", + "parameters": [ + { + "in": "path", + "name": "resource_type_id", + "required": true, + "schema": { + "title": "ResourceType ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Resource Type", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Schemas": { + "get": { + "description": "SCIM Schemas endpoint per RFC 7643 Section 7.\n\nReturns a ListResponse of all schemas supported by this service provider.", + "operationId": "get_schemas_scim_v2_Schemas_get", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Schemas", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Schemas/{schema_id}": { + "get": { + "description": "Get a single Schema by its URI per RFC 7643 Section 7.", + "operationId": "get_schema_scim_v2_Schemas__schema_id__get", + "parameters": [ + { + "in": "path", + "name": "schema_id", + "required": true, + "schema": { + "title": "Schema URI", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Schema", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/ServiceProviderConfig": { + "get": { + "description": "Return SCIM Service Provider Configuration.", + "operationId": "get_service_provider_config_scim_v2_ServiceProviderConfig_get", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMServiceProviderConfig" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Service Provider Config", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Users": { + "get": { + "description": "Get a list of users according to SCIM v2 protocol", + "operationId": "get_users_scim_v2_Users_get", + "parameters": [ + { + "in": "query", + "name": "startIndex", + "required": false, + "schema": { + "default": 1, + "minimum": 1, + "title": "Startindex", + "type": "integer" + } + }, + { + "in": "query", + "name": "count", + "required": false, + "schema": { + "default": 10, + "maximum": 100, + "minimum": 1, + "title": "Count", + "type": "integer" + } + }, + { + "in": "query", + "name": "filter", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Filter" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Users", + "tags": [ + "scim" + ] + }, + "post": { + "description": "Create a user according to SCIM v2 protocol", + "operationId": "create_user_scim_v2_Users_post", + "parameters": [ + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create User", + "tags": [ + "scim" + ] + } + }, + "/scim/v2/Users/{user_id}": { + "delete": { + "description": "Delete a user according to SCIM v2 protocol", + "operationId": "delete_user_scim_v2_Users__user_id__delete", + "parameters": [ + { + "in": "path", + "name": "user_id", + "required": true, + "schema": { + "title": "User ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete User", + "tags": [ + "scim" + ] + }, + "get": { + "description": "Get a single user by ID according to SCIM v2 protocol", + "operationId": "get_user_scim_v2_Users__user_id__get", + "parameters": [ + { + "in": "path", + "name": "user_id", + "required": true, + "schema": { + "title": "User ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get User", + "tags": [ + "scim" + ] + }, + "patch": { + "description": "Patch a user according to SCIM v2 protocol", + "operationId": "patch_user_scim_v2_Users__user_id__patch", + "parameters": [ + { + "in": "path", + "name": "user_id", + "required": true, + "schema": { + "title": "User ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMPatchOp" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Patch User", + "tags": [ + "scim" + ] + }, + "put": { + "description": "Update a user according to SCIM v2 protocol (full replacement)", + "operationId": "update_user_scim_v2_Users__user_id__put", + "parameters": [ + { + "in": "path", + "name": "user_id", + "required": true, + "schema": { + "title": "User ID", + "type": "string" + } + }, + { + "in": "query", + "name": "feature", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feature" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCIMUser" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update User", + "tags": [ + "scim" + ] + } + } + } + }, + "search_tools": { + "components": { + "schemas": { + "CreateSearchToolRequest": { + "properties": { + "search_tool": { + "$ref": "#/components/schemas/SearchTool" + } + }, + "required": [ + "search_tool" + ], + "title": "CreateSearchToolRequest", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ListSearchToolsResponse": { + "description": "Response model for listing search tools.", + "properties": { + "search_tools": { + "items": { + "$ref": "#/components/schemas/SearchToolInfoResponse" + }, + "title": "Search Tools", + "type": "array" + } + }, + "required": [ + "search_tools" + ], + "title": "ListSearchToolsResponse", + "type": "object" + }, + "SearchTool": { + "description": "Search tool configuration.\n\nExample:\n {\n \"search_tool_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"search_tool_name\": \"litellm-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-...\"\n },\n \"search_tool_info\": {\n \"description\": \"Perplexity search tool\"\n }\n }", + "properties": { + "created_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "litellm_params": { + "$ref": "#/components/schemas/SearchToolLiteLLMParams" + }, + "search_tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Search Tool Id" + }, + "search_tool_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Search Tool Info" + }, + "search_tool_name": { + "title": "Search Tool Name", + "type": "string" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "search_tool_name", + "litellm_params" + ], + "title": "SearchTool", + "type": "object" + }, + "SearchToolInfoResponse": { + "description": "Response model for search tool information.", + "properties": { + "created_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "is_from_config": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Is From Config" + }, + "litellm_params": { + "additionalProperties": true, + "title": "Litellm Params", + "type": "object" + }, + "search_tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Search Tool Id" + }, + "search_tool_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Search Tool Info" + }, + "search_tool_name": { + "title": "Search Tool Name", + "type": "string" + }, + "updated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "title": "SearchToolInfoResponse", + "type": "object" + }, + "SearchToolLiteLLMParams": { + "description": "LiteLLM params for search tools configuration.", + "properties": { + "api_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Base" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Key" + }, + "max_retries": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Retries" + }, + "search_provider": { + "title": "Search Provider", + "type": "string" + }, + "timeout": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Timeout" + } + }, + "required": [ + "search_provider" + ], + "title": "SearchToolLiteLLMParams", + "type": "object" + }, + "TestSearchToolConnectionRequest": { + "properties": { + "litellm_params": { + "additionalProperties": true, + "title": "Litellm Params", + "type": "object" + } + }, + "required": [ + "litellm_params" + ], + "title": "TestSearchToolConnectionRequest", + "type": "object" + }, + "UpdateSearchToolRequest": { + "properties": { + "search_tool": { + "$ref": "#/components/schemas/SearchTool" + } + }, + "required": [ + "search_tool" + ], + "title": "UpdateSearchToolRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/search_tools": { + "post": { + "description": "Create a new search tool.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/search_tools\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"search_tool\": {\n \"search_tool_name\": \"litellm-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-...\"\n },\n \"search_tool_info\": {\n \"description\": \"Perplexity search tool\"\n }\n }\n }'\n```\n\nExample Response:\n```json\n{\n \"search_tool_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"search_tool_name\": \"litellm-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-...\"\n },\n \"search_tool_info\": {\n \"description\": \"Perplexity search tool\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n}\n```", + "operationId": "create_search_tool_search_tools_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateSearchToolRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Search Tool", + "tags": [ + "search_tools" + ] + } + }, + "/search_tools/list": { + "get": { + "description": "List all search tools that are available in the database and config file.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/search_tools/list\" -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"search_tools\": [\n {\n \"search_tool_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"search_tool_name\": \"litellm-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-***\",\n \"api_base\": \"https://api.perplexity.ai\"\n },\n \"search_tool_info\": {\n \"description\": \"Perplexity search tool\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\",\n \"is_from_config\": false\n },\n {\n \"search_tool_name\": \"config-search-tool\",\n \"litellm_params\": {\n \"search_provider\": \"tavily\",\n \"api_key\": \"tvly-***\"\n },\n \"is_from_config\": true\n }\n ]\n}\n```", + "operationId": "list_search_tools_search_tools_list_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListSearchToolsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Search Tools", + "tags": [ + "search_tools" + ] + } + }, + "/search_tools/test_connection": { + "post": { + "description": "Test connection to a search provider with the given configuration.\n\nMakes a simple test search query to verify the API key and configuration are valid.\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/search_tools/test_connection\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-...\"\n }\n }'\n```\n\nExample Response (Success):\n```json\n{\n \"status\": \"success\",\n \"message\": \"Successfully connected to perplexity search provider\",\n \"test_query\": \"test\",\n \"results_count\": 5\n}\n```\n\nExample Response (Failure):\n```json\n{\n \"status\": \"error\",\n \"message\": \"Authentication failed: Invalid API key\",\n \"error_type\": \"AuthenticationError\"\n}\n```", + "operationId": "test_search_tool_connection_search_tools_test_connection_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TestSearchToolConnectionRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Test Search Tool Connection", + "tags": [ + "search_tools" + ] + } + }, + "/search_tools/ui/available_providers": { + "get": { + "description": "Get the list of available search providers with their configuration fields.\n\nAuto-discovers search providers and their UI-friendly names from transformation configs.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/search_tools/ui/available_providers\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"providers\": [\n {\n \"provider_name\": \"perplexity\",\n \"ui_friendly_name\": \"Perplexity\"\n },\n {\n \"provider_name\": \"tavily\",\n \"ui_friendly_name\": \"Tavily\"\n }\n ]\n}\n```", + "operationId": "get_available_search_providers_search_tools_ui_available_providers_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Available Search Providers", + "tags": [ + "search_tools" + ] + } + }, + "/search_tools/{search_tool_id}": { + "delete": { + "description": "Delete a search tool.\n\nExample Request:\n```bash\ncurl -X DELETE \"http://localhost:4000/search_tools/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"message\": \"Search tool 123e4567-e89b-12d3-a456-426614174000 deleted successfully\",\n \"search_tool_name\": \"litellm-search\"\n}\n```", + "operationId": "delete_search_tool_search_tools__search_tool_id__delete", + "parameters": [ + { + "in": "path", + "name": "search_tool_id", + "required": true, + "schema": { + "title": "Search Tool Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Search Tool", + "tags": [ + "search_tools" + ] + }, + "get": { + "description": "Get detailed information about a specific search tool by ID.\n\nExample Request:\n```bash\ncurl -X GET \"http://localhost:4000/search_tools/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \"\n```\n\nExample Response:\n```json\n{\n \"search_tool_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"search_tool_name\": \"litellm-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-***\"\n },\n \"search_tool_info\": {\n \"description\": \"Perplexity search tool\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T12:34:56.789Z\"\n}\n```", + "operationId": "get_search_tool_info_search_tools__search_tool_id__get", + "parameters": [ + { + "in": "path", + "name": "search_tool_id", + "required": true, + "schema": { + "title": "Search Tool Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Search Tool Info", + "tags": [ + "search_tools" + ] + }, + "put": { + "description": "Update an existing search tool.\n\nExample Request:\n```bash\ncurl -X PUT \"http://localhost:4000/search_tools/123e4567-e89b-12d3-a456-426614174000\" \\\n -H \"Authorization: Bearer \" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"search_tool\": {\n \"search_tool_name\": \"updated-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-new-key\"\n },\n \"search_tool_info\": {\n \"description\": \"Updated search tool\"\n }\n }\n }'\n```\n\nExample Response:\n```json\n{\n \"search_tool_id\": \"123e4567-e89b-12d3-a456-426614174000\",\n \"search_tool_name\": \"updated-search\",\n \"litellm_params\": {\n \"search_provider\": \"perplexity\",\n \"api_key\": \"sk-new-key\"\n },\n \"search_tool_info\": {\n \"description\": \"Updated search tool\"\n },\n \"created_at\": \"2023-11-09T12:34:56.789Z\",\n \"updated_at\": \"2023-11-09T13:45:12.345Z\"\n}\n```", + "operationId": "update_search_tool_search_tools__search_tool_id__put", + "parameters": [ + { + "in": "path", + "name": "search_tool_id", + "required": true, + "schema": { + "title": "Search Tool Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateSearchToolRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Search Tool", + "tags": [ + "search_tools" + ] + } + } + } + }, + "tools": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "LiteLLM_ToolTableRow": { + "properties": { + "assignments": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Assignments" + }, + "call_count": { + "default": 0, + "title": "Call Count", + "type": "integer" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, + "input_policy": { + "default": "untrusted", + "enum": [ + "trusted", + "untrusted", + "blocked" + ], + "title": "Input Policy", + "type": "string" + }, + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Alias" + }, + "key_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Hash" + }, + "last_used_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Used At" + }, + "origin": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Origin" + }, + "output_policy": { + "default": "untrusted", + "enum": [ + "trusted", + "untrusted" + ], + "title": "Output Policy", + "type": "string" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "tool_id": { + "title": "Tool Id", + "type": "string" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "updated_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated By" + }, + "user_agent": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Agent" + } + }, + "required": [ + "tool_id", + "tool_name" + ], + "title": "LiteLLM_ToolTableRow", + "type": "object" + }, + "ToolDetailResponse": { + "properties": { + "overrides": { + "items": { + "$ref": "#/components/schemas/ToolPolicyOverrideRow" + }, + "title": "Overrides", + "type": "array" + }, + "tool": { + "$ref": "#/components/schemas/LiteLLM_ToolTableRow" + } + }, + "required": [ + "tool" + ], + "title": "ToolDetailResponse", + "type": "object" + }, + "ToolListResponse": { + "properties": { + "tools": { + "items": { + "$ref": "#/components/schemas/LiteLLM_ToolTableRow" + }, + "title": "Tools", + "type": "array" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "tools", + "total" + ], + "title": "ToolListResponse", + "type": "object" + }, + "ToolPolicyOption": { + "properties": { + "description": { + "title": "Description", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "value", + "label", + "description" + ], + "title": "ToolPolicyOption", + "type": "object" + }, + "ToolPolicyOptionsResponse": { + "properties": { + "input_policies": { + "items": { + "$ref": "#/components/schemas/ToolPolicyOption" + }, + "title": "Input Policies", + "type": "array" + }, + "output_policies": { + "items": { + "$ref": "#/components/schemas/ToolPolicyOption" + }, + "title": "Output Policies", + "type": "array" + } + }, + "required": [ + "input_policies", + "output_policies" + ], + "title": "ToolPolicyOptionsResponse", + "type": "object" + }, + "ToolPolicyOverrideRow": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "input_policy": { + "default": "blocked", + "enum": [ + "trusted", + "untrusted", + "blocked" + ], + "title": "Input Policy", + "type": "string" + }, + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Alias" + }, + "key_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Hash" + }, + "override_id": { + "title": "Override Id", + "type": "string" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + } + }, + "required": [ + "override_id", + "tool_name" + ], + "title": "ToolPolicyOverrideRow", + "type": "object" + }, + "ToolPolicyUpdateRequest": { + "properties": { + "input_policy": { + "anyOf": [ + { + "enum": [ + "trusted", + "untrusted", + "blocked" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Input Policy" + }, + "key_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Alias" + }, + "key_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Hash" + }, + "output_policy": { + "anyOf": [ + { + "enum": [ + "trusted", + "untrusted" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Output Policy" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + } + }, + "required": [ + "tool_name" + ], + "title": "ToolPolicyUpdateRequest", + "type": "object" + }, + "ToolPolicyUpdateResponse": { + "properties": { + "input_policy": { + "anyOf": [ + { + "enum": [ + "trusted", + "untrusted", + "blocked" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Input Policy" + }, + "key_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key Hash" + }, + "output_policy": { + "anyOf": [ + { + "enum": [ + "trusted", + "untrusted" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Output Policy" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + }, + "updated": { + "title": "Updated", + "type": "boolean" + } + }, + "required": [ + "tool_name", + "updated" + ], + "title": "ToolPolicyUpdateResponse", + "type": "object" + }, + "ToolUsageLogEntry": { + "description": "One spend log row for a tool call (for UI \"recent logs\" table).", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "input_snippet": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Input Snippet" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "spend": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Spend" + }, + "timestamp": { + "title": "Timestamp", + "type": "string" + }, + "total_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Total Tokens" + } + }, + "required": [ + "id", + "timestamp" + ], + "title": "ToolUsageLogEntry", + "type": "object" + }, + "ToolUsageLogsResponse": { + "properties": { + "logs": { + "items": { + "$ref": "#/components/schemas/ToolUsageLogEntry" + }, + "title": "Logs", + "type": "array" + }, + "page": { + "title": "Page", + "type": "integer" + }, + "page_size": { + "title": "Page Size", + "type": "integer" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "logs", + "total", + "page", + "page_size" + ], + "title": "ToolUsageLogsResponse", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/toolset/{toolset_name}/mcp": { + "delete": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "get": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "head": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "options": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "patch": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "post": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + }, + "put": { + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", + "parameters": [ + { + "in": "path", + "name": "toolset_name", + "required": true, + "schema": { + "title": "Toolset Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Toolset Mcp Route", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/list": { + "get": { + "description": "List all auto-discovered tools and their policies.\n\nParameters:\n- input_policy: Optional filter \u2014 one of \"trusted\", \"untrusted\", \"blocked\"", + "operationId": "list_tools_v1_tool_list_get", + "parameters": [ + { + "in": "query", + "name": "input_policy", + "required": false, + "schema": { + "anyOf": [ + { + "enum": [ + "trusted", + "untrusted", + "blocked" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Input Policy" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Tools", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/policy": { + "post": { + "description": "Set the input_policy and/or output_policy for a tool (global), or block for a specific team/key (override).\n\nParameters:\n- tool_name: str - The tool to update\n- input_policy: optional - \"trusted\" | \"untrusted\" | \"blocked\"\n- output_policy: optional - \"trusted\" | \"untrusted\"\n- team_id: optional - if set, create/update override for this team only\n- key_hash: optional - if set, create/update override for this key only", + "operationId": "update_tool_policy_v1_tool_policy_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolPolicyUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolPolicyUpdateResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Tool Policy", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/policy/options": { + "get": { + "description": "Return the available input and output policy options with descriptions.\nStatic data \u2014 no DB call.", + "operationId": "get_tool_policy_options_v1_tool_policy_options_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolPolicyOptionsResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Tool Policy Options", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/{tool_name}": { + "get": { + "description": "Get details for a single tool.", + "operationId": "get_tool_v1_tool__tool_name__get", + "parameters": [ + { + "in": "path", + "name": "tool_name", + "required": true, + "schema": { + "title": "Tool Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_ToolTableRow" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Tool", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/{tool_name}/detail": { + "get": { + "description": "Get a single tool with its policy overrides (for UI detail view).", + "operationId": "get_tool_detail_v1_tool__tool_name__detail_get", + "parameters": [ + { + "in": "path", + "name": "tool_name", + "required": true, + "schema": { + "title": "Tool Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolDetailResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Tool Detail", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/{tool_name}/logs": { + "get": { + "description": "Return paginated spend logs for requests that used this tool (from SpendLogToolIndex).", + "operationId": "get_tool_usage_logs_v1_tool__tool_name__logs_get", + "parameters": [ + { + "in": "path", + "name": "tool_name", + "required": true, + "schema": { + "title": "Tool Name", + "type": "string" + } + }, + { + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "minimum": 1, + "title": "Page", + "type": "integer" + } + }, + { + "in": "query", + "name": "page_size", + "required": false, + "schema": { + "default": 50, + "maximum": 100, + "minimum": 1, + "title": "Page Size", + "type": "integer" + } + }, + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "Start Date" + } + }, + { + "description": "YYYY-MM-DD", + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD", + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolUsageLogsResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Tool Usage Logs", + "tags": [ + "tools" + ] + } + }, + "/v1/tool/{tool_name}/overrides": { + "delete": { + "description": "Remove a policy override for a tool. Specify the override by team_id or key_hash\n(exactly one required).", + "operationId": "delete_tool_policy_override_v1_tool__tool_name__overrides_delete", + "parameters": [ + { + "in": "path", + "name": "tool_name", + "required": true, + "schema": { + "title": "Tool Name", + "type": "string" + } + }, + { + "description": "Team ID of the override to remove", + "in": "query", + "name": "team_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Team ID of the override to remove", + "title": "Team Id" + } + }, + { + "description": "Key hash of the override to remove", + "in": "query", + "name": "key_hash", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Key hash of the override to remove", + "title": "Key Hash" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Tool Policy Override", + "tags": [ + "tools" + ] + } + } + } + }, + "usage_ai": { + "components": { + "schemas": { + "ChatMessage": { + "properties": { + "content": { + "title": "Content", + "type": "string" + }, + "role": { + "enum": [ + "user", + "assistant" + ], + "title": "Role", + "type": "string" + } + }, + "required": [ + "role", + "content" + ], + "title": "ChatMessage", + "type": "object" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "UsageAIChatRequest": { + "properties": { + "messages": { + "description": "Chat messages (user/assistant history)", + "items": { + "$ref": "#/components/schemas/ChatMessage" + }, + "title": "Messages", + "type": "array" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Model to use for AI chat", + "title": "Model" + } + }, + "required": [ + "messages" + ], + "title": "UsageAIChatRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/usage/ai/chat": { + "post": { + "description": "AI chat about usage data. Streams SSE events with the AI response.\nThe AI agent has access to tools that query aggregated daily activity data.", + "operationId": "usage_ai_chat_usage_ai_chat_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UsageAIChatRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Usage Ai Chat", + "tags": [ + "usage_ai" + ] + } + } + } + }, + "vantage": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + }, + "VantageDryRunRequest": { + "description": "Request model for Vantage dry-run operations (capped for preview)", + "properties": { + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 500, + "description": "Limit on number of records to preview (default: 500)", + "title": "Limit" + } + }, + "title": "VantageDryRunRequest", + "type": "object" + }, + "VantageExportRequest": { + "description": "Request model for Vantage export operations (actual export, no default limit)", + "properties": { + "end_time_utc": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "End time for data export in UTC", + "title": "End Time Utc" + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Optional limit on number of records to export (default: no limit)", + "title": "Limit" + }, + "start_time_utc": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Start time for data export in UTC", + "title": "Start Time Utc" + } + }, + "title": "VantageExportRequest", + "type": "object" + }, + "VantageExportResponse": { + "description": "Response model for Vantage export operations", + "properties": { + "dry_run_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Dry run data including usage data and FOCUS transformed data", + "title": "Dry Run Data" + }, + "message": { + "title": "Message", + "type": "string" + }, + "status": { + "title": "Status", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Summary statistics for dry run", + "title": "Summary" + } + }, + "required": [ + "message", + "status" + ], + "title": "VantageExportResponse", + "type": "object" + }, + "VantageInitRequest": { + "description": "Request model for initializing Vantage settings", + "properties": { + "api_key": { + "description": "Vantage API key for authentication", + "title": "Api Key", + "type": "string" + }, + "base_url": { + "default": "https://api.vantage.sh", + "description": "Vantage API base URL (default: https://api.vantage.sh)", + "title": "Base Url", + "type": "string" + }, + "integration_token": { + "description": "Vantage integration token for the cost-import endpoint", + "title": "Integration Token", + "type": "string" + } + }, + "required": [ + "api_key", + "integration_token" + ], + "title": "VantageInitRequest", + "type": "object" + }, + "VantageInitResponse": { + "description": "Response model for Vantage initialization", + "properties": { + "message": { + "title": "Message", + "type": "string" + }, + "status": { + "title": "Status", + "type": "string" + } + }, + "required": [ + "message", + "status" + ], + "title": "VantageInitResponse", + "type": "object" + }, + "VantageSettingsUpdate": { + "description": "Request model for updating Vantage settings", + "properties": { + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New Vantage API key for authentication", + "title": "Api Key" + }, + "base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New Vantage API base URL", + "title": "Base Url" + }, + "integration_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "New Vantage integration token", + "title": "Integration Token" + } + }, + "title": "VantageSettingsUpdate", + "type": "object" + }, + "VantageSettingsView": { + "description": "Response model for viewing Vantage settings with masked API key", + "properties": { + "api_key_masked": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Masked API key showing only first 4 and last 4 characters", + "title": "Api Key Masked" + }, + "base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Vantage API base URL", + "title": "Base Url" + }, + "integration_token_masked": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Masked integration token showing only first 4 and last 4 characters", + "title": "Integration Token Masked" + }, + "status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Configuration status", + "title": "Status" + } + }, + "title": "VantageSettingsView", + "type": "object" + } + } + }, + "paths": { + "/vantage/delete": { + "delete": { + "description": "Delete Vantage settings from the database.\n\nOnly admin users can delete Vantage settings.", + "operationId": "delete_vantage_settings_vantage_delete_delete", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageInitResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Vantage Settings", + "tags": [ + "vantage" + ] + } + }, + "/vantage/dry-run": { + "post": { + "description": "Perform a dry run export using the Vantage logger.\n\nReturns the data that would be exported without actually sending it to Vantage.\n\nParameters:\n- limit: Limit on number of records to preview (default: 500)\n\nOnly admin users can perform Vantage exports.", + "operationId": "vantage_dry_run_export_vantage_dry_run_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageDryRunRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageExportResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vantage Dry Run Export", + "tags": [ + "vantage" + ] + } + }, + "/vantage/export": { + "post": { + "description": "Perform an actual export using the Vantage logger.\n\nExports usage data in FOCUS CSV format to the Vantage API.\n\nParameters:\n- limit: Optional limit on number of records to export\n- start_time_utc: Optional start time for data export\n- end_time_utc: Optional end time for data export\n\nOnly admin users can perform Vantage exports.", + "operationId": "vantage_export_vantage_export_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageExportRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageExportResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vantage Export", + "tags": [ + "vantage" + ] + } + }, + "/vantage/init": { + "post": { + "description": "Initialize Vantage settings and store in the database.\n\nParameters:\n- api_key: Vantage API key for authentication\n- integration_token: Vantage integration token for the cost-import endpoint\n- base_url: Vantage API base URL (default: https://api.vantage.sh)\n\nOnly admin users can configure Vantage settings.", + "operationId": "init_vantage_settings_vantage_init_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageInitRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageInitResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Init Vantage Settings", + "tags": [ + "vantage" + ] + } + }, + "/vantage/settings": { + "get": { + "description": "View current Vantage settings.\n\nReturns the current Vantage configuration with the API key masked for security.\nOnly admin users can view Vantage settings.", + "operationId": "get_vantage_settings_vantage_settings_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageSettingsView" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Vantage Settings", + "tags": [ + "vantage" + ] + }, + "put": { + "description": "Update existing Vantage settings.\n\nAllows updating individual Vantage configuration fields without requiring all fields.\nOnly admin users can update Vantage settings.", + "operationId": "update_vantage_settings_vantage_settings_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageSettingsUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VantageInitResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Vantage Settings", + "tags": [ + "vantage" + ] + } + } + } + }, + "vector_store_files": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/v1/vector_stores": { + "get": { + "description": "List vector stores.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/list", + "operationId": "vector_store_list_v1_vector_stores_get", + "parameters": [ + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "desc", + "title": "Order" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store List", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "description": "Create a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/create\n\nSupports target_model_names parameter for creating vector stores across multiple models:\n```json\n{\n \"name\": \"my-vector-store\",\n \"target_model_names\": \"gpt-4,gemini-2.0\"\n}\n```", + "operationId": "vector_store_create_v1_vector_stores_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Create", + "tags": [ + "vector_store_files" + ] + } + }, + "/v1/vector_stores/{vector_store_id}": { + "delete": { + "description": "Delete a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/delete", + "operationId": "vector_store_delete_v1_vector_stores__vector_store_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Delete", + "tags": [ + "vector_store_files" + ] + }, + "get": { + "description": "Retrieve a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/retrieve", + "operationId": "vector_store_retrieve_v1_vector_stores__vector_store_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Retrieve", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "description": "Update a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/modify", + "operationId": "vector_store_update_v1_vector_stores__vector_store_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Update", + "tags": [ + "vector_store_files" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files": { + "get": { + "operationId": "vector_store_file_list_v1_vector_stores__vector_store_id__files_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File List", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "operationId": "vector_store_file_create_v1_vector_stores__vector_store_id__files_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Create", + "tags": [ + "vector_store_files" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files/{file_id}": { + "delete": { + "operationId": "vector_store_file_delete_v1_vector_stores__vector_store_id__files__file_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Delete", + "tags": [ + "vector_store_files" + ] + }, + "get": { + "operationId": "vector_store_file_retrieve_v1_vector_stores__vector_store_id__files__file_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Retrieve", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "operationId": "vector_store_file_update_v1_vector_stores__vector_store_id__files__file_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Update", + "tags": [ + "vector_store_files" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files/{file_id}/content": { + "get": { + "operationId": "vector_store_file_content_v1_vector_stores__vector_store_id__files__file_id__content_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Content", + "tags": [ + "vector_store_files" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/search": { + "post": { + "description": "Search a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/search", + "operationId": "vector_store_search_v1_vector_stores__vector_store_id__search_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Search", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores": { + "get": { + "description": "List vector stores.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/list", + "operationId": "vector_store_list_vector_stores_get", + "parameters": [ + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "desc", + "title": "Order" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store List", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "description": "Create a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/create\n\nSupports target_model_names parameter for creating vector stores across multiple models:\n```json\n{\n \"name\": \"my-vector-store\",\n \"target_model_names\": \"gpt-4,gemini-2.0\"\n}\n```", + "operationId": "vector_store_create_vector_stores_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Create", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores/{vector_store_id}": { + "delete": { + "description": "Delete a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/delete", + "operationId": "vector_store_delete_vector_stores__vector_store_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Delete", + "tags": [ + "vector_store_files" + ] + }, + "get": { + "description": "Retrieve a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/retrieve", + "operationId": "vector_store_retrieve_vector_stores__vector_store_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Retrieve", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "description": "Update a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/modify", + "operationId": "vector_store_update_vector_stores__vector_store_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Update", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores/{vector_store_id}/files": { + "get": { + "operationId": "vector_store_file_list_vector_stores__vector_store_id__files_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File List", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "operationId": "vector_store_file_create_vector_stores__vector_store_id__files_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Create", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores/{vector_store_id}/files/{file_id}": { + "delete": { + "operationId": "vector_store_file_delete_vector_stores__vector_store_id__files__file_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Delete", + "tags": [ + "vector_store_files" + ] + }, + "get": { + "operationId": "vector_store_file_retrieve_vector_stores__vector_store_id__files__file_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Retrieve", + "tags": [ + "vector_store_files" + ] + }, + "post": { + "operationId": "vector_store_file_update_vector_stores__vector_store_id__files__file_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Update", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores/{vector_store_id}/files/{file_id}/content": { + "get": { + "operationId": "vector_store_file_content_vector_stores__vector_store_id__files__file_id__content_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Content", + "tags": [ + "vector_store_files" + ] + } + }, + "/vector_stores/{vector_store_id}/search": { + "post": { + "description": "Search a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/search", + "operationId": "vector_store_search_vector_stores__vector_store_id__search_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Search", + "tags": [ + "vector_store_files" + ] + } + } + } + }, + "vector_store_management": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "LiteLLM_ManagedVectorStore": { + "description": "LiteLLM managed vector store object - this is is the object stored in the database", + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "custom_llm_provider": { + "title": "Custom Llm Provider", + "type": "string" + }, + "litellm_credential_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Litellm Credential Name" + }, + "litellm_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Litellm Params" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Id" + }, + "vector_store_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Description" + }, + "vector_store_id": { + "title": "Vector Store Id", + "type": "string" + }, + "vector_store_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Metadata" + }, + "vector_store_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Name" + } + }, + "title": "LiteLLM_ManagedVectorStore", + "type": "object" + }, + "LiteLLM_ManagedVectorStoreListResponse": { + "description": "Response format for listing vector stores", + "properties": { + "current_page": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Current Page" + }, + "data": { + "items": { + "$ref": "#/components/schemas/LiteLLM_ManagedVectorStore" + }, + "title": "Data", + "type": "array" + }, + "object": { + "const": "list", + "title": "Object", + "type": "string" + }, + "total_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Total Count" + }, + "total_pages": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Total Pages" + } + }, + "title": "LiteLLM_ManagedVectorStoreListResponse", + "type": "object" + }, + "LiteLLM_ManagedVectorStoresTable": { + "properties": { + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "custom_llm_provider": { + "title": "Custom Llm Provider", + "type": "string" + }, + "litellm_credential_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Litellm Credential Name" + }, + "litellm_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Litellm Params" + }, + "team_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Team Id" + }, + "updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Updated At" + }, + "user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Id" + }, + "vector_store_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Description" + }, + "vector_store_id": { + "title": "Vector Store Id", + "type": "string" + }, + "vector_store_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Vector Store Metadata" + }, + "vector_store_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Name" + } + }, + "required": [ + "vector_store_id", + "custom_llm_provider", + "vector_store_name", + "vector_store_description", + "vector_store_metadata", + "created_at", + "updated_at", + "litellm_credential_name", + "litellm_params", + "team_id", + "user_id" + ], + "title": "LiteLLM_ManagedVectorStoresTable", + "type": "object" + }, + "ResponseLiteLLM_ManagedVectorStore": { + "properties": { + "vector_store": { + "$ref": "#/components/schemas/LiteLLM_ManagedVectorStoresTable" + } + }, + "title": "ResponseLiteLLM_ManagedVectorStore", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + }, + "VectorStoreDeleteRequest": { + "properties": { + "vector_store_id": { + "title": "Vector Store Id", + "type": "string" + } + }, + "required": [ + "vector_store_id" + ], + "title": "VectorStoreDeleteRequest", + "type": "object" + }, + "VectorStoreInfoRequest": { + "properties": { + "vector_store_id": { + "title": "Vector Store Id", + "type": "string" + } + }, + "required": [ + "vector_store_id" + ], + "title": "VectorStoreInfoRequest", + "type": "object" + }, + "VectorStoreUpdateRequest": { + "properties": { + "custom_llm_provider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Custom Llm Provider" + }, + "vector_store_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Description" + }, + "vector_store_id": { + "title": "Vector Store Id", + "type": "string" + }, + "vector_store_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Vector Store Metadata" + }, + "vector_store_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector Store Name" + } + }, + "required": [ + "vector_store_id" + ], + "title": "VectorStoreUpdateRequest", + "type": "object" + } + } + }, + "paths": { + "/v1/vector_store/list": { + "get": { + "description": "List all available vector stores with optional filtering and pagination.\nCombines both in-memory vector stores and those stored in the database.\nDatabase is the source of truth - deleted stores are removed from memory, updated stores sync to memory.\n\nParameters:\n- page: int - Page number for pagination (default: 1)\n- page_size: int - Number of items per page (default: 100)", + "operationId": "list_vector_stores_v1_vector_store_list_get", + "parameters": [ + { + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "title": "Page", + "type": "integer" + } + }, + { + "in": "query", + "name": "page_size", + "required": false, + "schema": { + "default": 100, + "title": "Page Size", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_ManagedVectorStoreListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Vector Stores", + "tags": [ + "vector_store_management" + ] + } + }, + "/vector_store/delete": { + "post": { + "description": "Delete a vector store from both database and in-memory registry.\n\nParameters:\n- vector_store_id: str - ID of the vector store to delete", + "operationId": "delete_vector_store_vector_store_delete_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VectorStoreDeleteRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Delete Vector Store", + "tags": [ + "vector_store_management" + ] + } + }, + "/vector_store/info": { + "post": { + "description": "Return a single vector store's details", + "operationId": "get_vector_store_info_vector_store_info_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VectorStoreInfoRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResponseLiteLLM_ManagedVectorStore" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Vector Store Info", + "tags": [ + "vector_store_management" + ] + } + }, + "/vector_store/list": { + "get": { + "description": "List all available vector stores with optional filtering and pagination.\nCombines both in-memory vector stores and those stored in the database.\nDatabase is the source of truth - deleted stores are removed from memory, updated stores sync to memory.\n\nParameters:\n- page: int - Page number for pagination (default: 1)\n- page_size: int - Number of items per page (default: 100)", + "operationId": "list_vector_stores_vector_store_list_get", + "parameters": [ + { + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "title": "Page", + "type": "integer" + } + }, + { + "in": "query", + "name": "page_size", + "required": false, + "schema": { + "default": 100, + "title": "Page Size", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_ManagedVectorStoreListResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "List Vector Stores", + "tags": [ + "vector_store_management" + ] + } + }, + "/vector_store/new": { + "post": { + "description": "Create a new vector store.\n\nParameters:\n- vector_store_id: str - Unique identifier for the vector store\n- custom_llm_provider: str - Provider of the vector store\n- vector_store_name: Optional[str] - Name of the vector store\n- vector_store_description: Optional[str] - Description of the vector store\n- vector_store_metadata: Optional[Dict] - Additional metadata for the vector store", + "operationId": "new_vector_store_vector_store_new_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LiteLLM_ManagedVectorStore" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "New Vector Store", + "tags": [ + "vector_store_management" + ] + } + }, + "/vector_store/update": { + "post": { + "description": "Update vector store details in both database and in-memory registry.\nThe updated data is immediately synchronized to the in-memory registry.", + "operationId": "update_vector_store_vector_store_update_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VectorStoreUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Update Vector Store", + "tags": [ + "vector_store_management" + ] + } + } + } + }, + "vector_stores": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "IndexCreateLiteLLMParams": { + "properties": { + "vector_store_index": { + "title": "Vector Store Index", + "type": "string" + }, + "vector_store_name": { + "title": "Vector Store Name", + "type": "string" + } + }, + "required": [ + "vector_store_index", + "vector_store_name" + ], + "title": "IndexCreateLiteLLMParams", + "type": "object" + }, + "IndexCreateRequest": { + "properties": { + "index_info": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Index Info" + }, + "index_name": { + "title": "Index Name", + "type": "string" + }, + "litellm_params": { + "$ref": "#/components/schemas/IndexCreateLiteLLMParams" + } + }, + "required": [ + "index_name", + "litellm_params" + ], + "title": "IndexCreateRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/v1/indexes": { + "post": { + "description": "Create an index. Just writes the index to the database.\n\n```bash\ncurl -L -X POST 'http://0.0.0.0:4000/indexes/create' -H 'Content-Type: application/json' -H 'Authorization: Bearer sk-1234' -H 'LiteLLM-Beta: indexes_beta=v1' -d '{ \n \"index_name\": \"dall-e-3\",\n \"vector_store_index\": \"real-index-name\",\n \"vector_store_name\": \"azure-ai-search\"\n }'\n```", + "operationId": "index_create_v1_indexes_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IndexCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Index Create", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores": { + "get": { + "description": "List vector stores.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/list", + "operationId": "vector_store_list_v1_vector_stores_get", + "parameters": [ + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "desc", + "title": "Order" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store List", + "tags": [ + "vector_stores" + ] + }, + "post": { + "description": "Create a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/create\n\nSupports target_model_names parameter for creating vector stores across multiple models:\n```json\n{\n \"name\": \"my-vector-store\",\n \"target_model_names\": \"gpt-4,gemini-2.0\"\n}\n```", + "operationId": "vector_store_create_v1_vector_stores_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Create", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores/{vector_store_id}": { + "delete": { + "description": "Delete a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/delete", + "operationId": "vector_store_delete_v1_vector_stores__vector_store_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Delete", + "tags": [ + "vector_stores" + ] + }, + "get": { + "description": "Retrieve a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/retrieve", + "operationId": "vector_store_retrieve_v1_vector_stores__vector_store_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Retrieve", + "tags": [ + "vector_stores" + ] + }, + "post": { + "description": "Update a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/modify", + "operationId": "vector_store_update_v1_vector_stores__vector_store_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Update", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files": { + "get": { + "operationId": "vector_store_file_list_v1_vector_stores__vector_store_id__files_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File List", + "tags": [ + "vector_stores" + ] + }, + "post": { + "operationId": "vector_store_file_create_v1_vector_stores__vector_store_id__files_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Create", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files/{file_id}": { + "delete": { + "operationId": "vector_store_file_delete_v1_vector_stores__vector_store_id__files__file_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Delete", + "tags": [ + "vector_stores" + ] + }, + "get": { + "operationId": "vector_store_file_retrieve_v1_vector_stores__vector_store_id__files__file_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Retrieve", + "tags": [ + "vector_stores" + ] + }, + "post": { + "operationId": "vector_store_file_update_v1_vector_stores__vector_store_id__files__file_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Update", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/files/{file_id}/content": { + "get": { + "operationId": "vector_store_file_content_v1_vector_stores__vector_store_id__files__file_id__content_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Content", + "tags": [ + "vector_stores" + ] + } + }, + "/v1/vector_stores/{vector_store_id}/search": { + "post": { + "description": "Search a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/search", + "operationId": "vector_store_search_v1_vector_stores__vector_store_id__search_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Search", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores": { + "get": { + "description": "List vector stores.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/list", + "operationId": "vector_store_list_vector_stores_get", + "parameters": [ + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "before", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Before" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 20, + "title": "Limit" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "desc", + "title": "Order" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store List", + "tags": [ + "vector_stores" + ] + }, + "post": { + "description": "Create a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/create\n\nSupports target_model_names parameter for creating vector stores across multiple models:\n```json\n{\n \"name\": \"my-vector-store\",\n \"target_model_names\": \"gpt-4,gemini-2.0\"\n}\n```", + "operationId": "vector_store_create_vector_stores_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Create", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores/{vector_store_id}": { + "delete": { + "description": "Delete a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/delete", + "operationId": "vector_store_delete_vector_stores__vector_store_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Delete", + "tags": [ + "vector_stores" + ] + }, + "get": { + "description": "Retrieve a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/retrieve", + "operationId": "vector_store_retrieve_vector_stores__vector_store_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Retrieve", + "tags": [ + "vector_stores" + ] + }, + "post": { + "description": "Update a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/modify", + "operationId": "vector_store_update_vector_stores__vector_store_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Update", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores/{vector_store_id}/files": { + "get": { + "operationId": "vector_store_file_list_vector_stores__vector_store_id__files_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File List", + "tags": [ + "vector_stores" + ] + }, + "post": { + "operationId": "vector_store_file_create_vector_stores__vector_store_id__files_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Create", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores/{vector_store_id}/files/{file_id}": { + "delete": { + "operationId": "vector_store_file_delete_vector_stores__vector_store_id__files__file_id__delete", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Delete", + "tags": [ + "vector_stores" + ] + }, + "get": { + "operationId": "vector_store_file_retrieve_vector_stores__vector_store_id__files__file_id__get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Retrieve", + "tags": [ + "vector_stores" + ] + }, + "post": { + "operationId": "vector_store_file_update_vector_stores__vector_store_id__files__file_id__post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Update", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores/{vector_store_id}/files/{file_id}/content": { + "get": { + "operationId": "vector_store_file_content_vector_stores__vector_store_id__files__file_id__content_get", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + }, + { + "in": "path", + "name": "file_id", + "required": true, + "schema": { + "title": "File Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store File Content", + "tags": [ + "vector_stores" + ] + } + }, + "/vector_stores/{vector_store_id}/search": { + "post": { + "description": "Search a vector store.\n\nAPI Reference:\nhttps://platform.openai.com/docs/api-reference/vector-stores/search", + "operationId": "vector_store_search_vector_stores__vector_store_id__search_post", + "parameters": [ + { + "in": "path", + "name": "vector_store_id", + "required": true, + "schema": { + "title": "Vector Store Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Vector Store Search", + "tags": [ + "vector_stores" + ] + } + } + } + } +} diff --git a/litellm/proxy/_lazy_openapi_snapshot.py b/litellm/proxy/_lazy_openapi_snapshot.py new file mode 100644 index 00000000000..309a0276aac --- /dev/null +++ b/litellm/proxy/_lazy_openapi_snapshot.py @@ -0,0 +1,106 @@ +""" +Per-feature OpenAPI snapshot for lazy-loaded routers. + +The committed JSON is generated by `python -m litellm.proxy._lazy_openapi_snapshot` +and consumed at runtime so /openapi.json can show full route info for unloaded +features without importing them. CI verifies the file is current and surfaces +any drift as a neutral check. +""" + +import json +import sys +from pathlib import Path +from typing import Dict, Optional + +SNAPSHOT_FILE = Path(__file__).parent / "_lazy_openapi_snapshot.json" +HTTP_METHODS = {"delete", "get", "head", "options", "patch", "post", "put"} + + +def load_snapshot() -> Optional[Dict[str, Dict]]: + if not SNAPSHOT_FILE.exists(): + return None + try: + with SNAPSHOT_FILE.open() as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return None + + +def _normalize_operation_ids(paths: Dict[str, Dict]) -> None: + """Make FastAPI-generated operation IDs stable for multi-method routes. + + FastAPI derives the default operation ID suffix from the first item in the + route's methods set. For routes registered with several HTTP methods, that + set iteration order can vary between processes, which makes the snapshot + drift even when no routes changed. + """ + for path_ops in paths.values(): + if not isinstance(path_ops, dict): + continue + + methods = {method for method in path_ops if method in HTTP_METHODS} + if not methods: + continue + + for method, operation in path_ops.items(): + if method not in HTTP_METHODS or not isinstance(operation, dict): + continue + + operation_id = operation.get("operationId") + if not isinstance(operation_id, str): + continue + + for suffix in methods: + suffix_token = f"_{suffix}" + if operation_id.endswith(suffix_token): + operation["operationId"] = ( + operation_id[: -len(suffix_token)] + f"_{method}" + ) + break + + +def generate_snapshot() -> Dict[str, Dict]: + import importlib + + from fastapi.openapi.utils import get_openapi + + from litellm.proxy._lazy_features import LAZY_FEATURES + from litellm.proxy.proxy_server import app + + for feat in LAZY_FEATURES: + if feat.module_path in sys.modules: + continue + try: + module = importlib.import_module(feat.module_path) + feat.register_fn(app, module) + except Exception as exc: + sys.stderr.write(f"warning: skip {feat.name}: {exc}\n") + + fragments: Dict[str, Dict] = {} + for feat in LAZY_FEATURES: + feat_routes = [ + r + for r in app.routes + if any(getattr(r, "path", "").startswith(p) for p in feat.path_prefixes) + ] + if not feat_routes: + continue + full = get_openapi(title=app.title, version=app.version, routes=feat_routes) + paths = full.get("paths", {}) + _normalize_operation_ids(paths) + # Group all of a feature's routes under one tag. + for path_ops in paths.values(): + for op in path_ops.values(): + if isinstance(op, dict): + op["tags"] = [feat.name] + fragments[feat.name] = { + "paths": paths, + "components": {"schemas": full.get("components", {}).get("schemas", {})}, + } + return fragments + + +if __name__ == "__main__": + fragments = generate_snapshot() + SNAPSHOT_FILE.write_text(json.dumps(fragments, indent=2, sort_keys=True) + "\n") + sys.stdout.write(f"wrote {len(fragments)} feature fragments to {SNAPSHOT_FILE}\n") diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index fe1ac8c2d52..e44df111a5f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -17,6 +17,9 @@ from typing_extensions import Required, TypedDict from litellm._uuid import uuid from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + validate_no_callback_env_reference, +) from litellm.types.integrations.slack_alerting import AlertType from litellm.types.llms.openai import ( AllMessageValues, @@ -665,6 +668,8 @@ class LiteLLMRoutes(enum.Enum): "/models/{model_id}", "/guardrails/list", "/v2/guardrails/list", + "/project/list", + "/project/info", ] + spend_tracking_routes + key_management_routes @@ -689,6 +694,9 @@ class LiteLLMRoutes(enum.Enum): "/model/{model_id}/update", "/prompt/list", "/prompt/info", + # Project read routes - endpoint scopes results to caller's teams (non-admin) + "/project/list", + "/project/info", # Invitation routes - org/team admins checked in endpoint via _user_has_admin_privileges "/invitation/new", "/invitation/delete", @@ -956,6 +964,7 @@ class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase): agents: Optional[List[str]] = None agent_access_groups: Optional[List[str]] = None models: Optional[List[str]] = None + search_tools: Optional[List[str]] = None class BudgetLimitEntry(LiteLLMPydanticObjectBase): @@ -1920,8 +1929,10 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): raise ValueError( f"Invalid callback variable: {key}. Must be one of {valid_keys}" ) - if not isinstance(value, str): - callback_vars[key] = str(value) + callback_vars[key] = str(value) + validate_no_callback_env_reference( + key, callback_vars[key], source="key/team callback metadata" + ) return values @@ -1986,6 +1997,7 @@ class LiteLLM_ObjectPermissionTable(LiteLLMPydanticObjectBase): agent_access_groups: Optional[List[str]] = [] mcp_toolsets: Optional[List[str]] = None blocked_tools: Optional[List[str]] = [] + search_tools: Optional[List[str]] = [] class LiteLLM_TeamTable(TeamBase): @@ -2206,8 +2218,8 @@ class PassThroughGenericEndpoint(LiteLLMPydanticObjectBase): description="The USD cost per request to the target endpoint. This is used to calculate the cost of the request to the target endpoint.", ) auth: bool = Field( - default=False, - description="Whether authentication is required for the pass-through endpoint. If True, requests to the endpoint will require a valid LiteLLM API key.", + default=True, + description="Whether authentication is required for the pass-through endpoint. Defaults to True so a pass-through silently created without an explicit value still requires a valid LiteLLM API key — set to False only if the endpoint is meant to be a public forwarder (e.g. an unauthenticated webhook target).", ) guardrails: Optional[PassThroughGuardrailsConfig] = Field( default=None, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 840f64cfede..113a8f538c0 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -12,14 +12,13 @@ Run checks for: import asyncio import re import time -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Type, Union, cast from fastapi import HTTPException, Request, status from pydantic import BaseModel import litellm from litellm._logging import verbose_proxy_logger -from litellm.caching.caching import DualCache from litellm.caching.dual_cache import LimitedSizeOrderedDict from litellm.constants import ( CLI_JWT_EXPIRATION_HOURS, @@ -66,6 +65,8 @@ from litellm.proxy.guardrails.tool_name_extraction import ( TOOL_CAPABLE_CALL_TYPES, extract_request_tool_names, ) +from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.route_llm_request import route_request from litellm.proxy.utils import PrismaClient, ProxyLogging, log_db_metrics from litellm.router import Router @@ -374,7 +375,7 @@ def _guardrail_modification_check( coerced = _coerce_to_dict(container) if coerced is None: return False - return any(coerced.get(key) for key in _GUARDRAIL_MODIFICATION_KEYS) + return any(key in coerced for key in _GUARDRAIL_MODIFICATION_KEYS) # Check both metadata keys — callers can populate either depending on the # endpoint. Cover the top-level too so root-level injection is rejected. @@ -852,7 +853,7 @@ def get_actual_routes(allowed_routes: list) -> list: async def get_default_end_user_budget( prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span] = None, ) -> Optional[LiteLLM_BudgetTable]: """ @@ -875,9 +876,12 @@ async def get_default_end_user_budget( cache_key = f"default_end_user_budget:{litellm.max_end_user_budget_id}" # Check cache first - cached_budget = await user_api_key_cache.async_get_cache(key=cache_key) + cached_budget = await user_api_key_cache.async_get_cache( + key=cache_key, + model_type=LiteLLM_BudgetTable, + ) if cached_budget is not None: - return LiteLLM_BudgetTable(**cached_budget) + return cached_budget # Fetch from database try: @@ -891,14 +895,16 @@ async def get_default_end_user_budget( ) return None + _budget_obj = LiteLLM_BudgetTable(**budget_record.dict()) # Cache the budget for 60 seconds await user_api_key_cache.async_set_cache( key=cache_key, - value=budget_record.dict(), + value=_budget_obj, + model_type=LiteLLM_BudgetTable, ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, ) - return LiteLLM_BudgetTable(**budget_record.dict()) + return _budget_obj except Exception as e: verbose_proxy_logger.error(f"Error fetching default end user budget: {str(e)}") @@ -909,13 +915,14 @@ async def get_default_end_user_budget( async def get_team_member_default_budget( budget_id: str, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, ) -> Optional[LiteLLM_BudgetTable]: """ Fetches the team-level default per-member budget referenced by team.metadata["team_member_budget_id"]. This budget is applied to team members whose TeamMembership row has no - linked budget. Results are cached for performance. + linked budget, or whose linked budget has max_budget=NULL. Results are + cached for performance. Args: budget_id: The budget_id pulled from team.metadata["team_member_budget_id"] @@ -965,7 +972,7 @@ async def get_team_member_default_budget( async def _apply_default_budget_to_end_user( end_user_obj: LiteLLM_EndUserTable, prisma_client: PrismaClient, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span] = None, ) -> LiteLLM_EndUserTable: """ @@ -1038,7 +1045,7 @@ def _check_end_user_budget( async def get_end_user_object( end_user_id: Optional[str], prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, route: str, parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, @@ -1069,10 +1076,12 @@ async def get_end_user_object( _key = "end_user_id:{}".format(end_user_id) # Check cache first - cached_user_obj = await user_api_key_cache.async_get_cache(key=_key) + cached_user_obj = await user_api_key_cache.async_get_cache( + key=_key, + model_type=LiteLLM_EndUserTable, + ) if cached_user_obj is not None: - return_obj = LiteLLM_EndUserTable(**cached_user_obj) - + return_obj = cached_user_obj # Apply default budget if needed return_obj = await _apply_default_budget_to_end_user( end_user_obj=return_obj, @@ -1107,9 +1116,11 @@ async def get_end_user_object( parent_otel_span=parent_otel_span, ) - # Save to cache (always store as dict for consistency) + # Save to cache await user_api_key_cache.async_set_cache( - key="end_user_id:{}".format(end_user_id), value=_response.dict() + key="end_user_id:{}".format(end_user_id), + value=_response, + model_type=LiteLLM_EndUserTable, ) # Check budget limits @@ -1127,7 +1138,7 @@ async def get_end_user_object( async def get_tag_objects_batch( tag_names: List[str], prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> Dict[str, LiteLLM_TagTable]: @@ -1160,12 +1171,12 @@ async def get_tag_objects_batch( # Try to get all tags from cache first for tag_name in tag_names: cache_key = f"tag:{tag_name}" - cached_tag = await user_api_key_cache.async_get_cache(key=cache_key) + cached_tag = await user_api_key_cache.async_get_cache( + key=cache_key, + model_type=LiteLLM_TagTable, + ) if cached_tag is not None: - if isinstance(cached_tag, dict): - tag_objects[tag_name] = LiteLLM_TagTable(**cached_tag) - else: - tag_objects[tag_name] = cached_tag + tag_objects[tag_name] = cached_tag else: uncached_tags.append(tag_name) @@ -1181,11 +1192,13 @@ async def get_tag_objects_batch( for db_tag in db_tags: tag_name = db_tag.tag_name cache_key = f"tag:{tag_name}" - # Cache with default TTL (same as end_user objects) + _tag_obj = LiteLLM_TagTable(**db_tag.dict()) await user_api_key_cache.async_set_cache( - key=cache_key, value=db_tag.dict() + key=cache_key, + value=_tag_obj, + model_type=LiteLLM_TagTable, ) - tag_objects[tag_name] = LiteLLM_TagTable(**db_tag.dict()) + tag_objects[tag_name] = _tag_obj except Exception as e: verbose_proxy_logger.debug(f"Error batch fetching tags from database: {e}") @@ -1196,7 +1209,7 @@ async def get_tag_objects_batch( async def get_tag_object( tag_name: Optional[str], prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> Optional[LiteLLM_TagTable]: @@ -1235,7 +1248,7 @@ async def get_team_membership( user_id: str, team_id: str, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> Optional["LiteLLM_TeamMembership"]: @@ -1255,9 +1268,12 @@ async def get_team_membership( _key = "team_membership:{}:{}".format(user_id, team_id) # check if in cache - cached_membership_obj = await user_api_key_cache.async_get_cache(key=_key) + cached_membership_obj = await user_api_key_cache.async_get_cache( + key=_key, + model_type=LiteLLM_TeamMembership, + ) if cached_membership_obj is not None: - return LiteLLM_TeamMembership(**cached_membership_obj) + return cached_membership_obj # else, check db try: @@ -1269,10 +1285,12 @@ async def get_team_membership( if response is None: return None - # save the team membership object to cache (store as dict) - await user_api_key_cache.async_set_cache(key=_key, value=response.dict()) - _response = LiteLLM_TeamMembership(**response.dict()) + await user_api_key_cache.async_set_cache( + key=_key, + value=_response, + model_type=LiteLLM_TeamMembership, + ) return _response except Exception: @@ -1440,7 +1458,7 @@ async def _get_fuzzy_user_object( async def get_user_object( user_id: Optional[str], prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, user_id_upsert: bool, parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, @@ -1459,12 +1477,12 @@ async def get_user_object( # check if in cache if not check_db_only: - cached_user_obj = await user_api_key_cache.async_get_cache(key=user_id) + cached_user_obj = await user_api_key_cache.async_get_cache( + key=user_id, + model_type=LiteLLM_UserTable, + ) if cached_user_obj is not None: - if isinstance(cached_user_obj, dict): - return LiteLLM_UserTable(**cached_user_obj) - elif isinstance(cached_user_obj, LiteLLM_UserTable): - return cached_user_obj + return cached_user_obj # else, check db if prisma_client is None: raise Exception("No db connected") @@ -1526,7 +1544,8 @@ async def get_user_object( # save the user object to cache await user_api_key_cache.async_set_cache( key=user_id, - value=response_dict, + value=_response, + model_type=LiteLLM_UserTable, ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, ) @@ -1547,13 +1566,21 @@ async def get_user_object( async def _cache_management_object( key: str, - value: BaseModel, - user_api_key_cache: DualCache, + value: Union[BaseModel, Dict[str, Any]], + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: Optional[ProxyLogging], + *, + model_type: Type[BaseModel], ): + """ + Persist management objects via ``UserApiKeyCache`` (in-memory + optional Redis). + + ``UserApiKeyCache`` serializes with ``model_type`` so Redis and in-memory stay aligned. + """ await user_api_key_cache.async_set_cache( key=key, value=value, + model_type=model_type, ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, ) @@ -1561,7 +1588,7 @@ async def _cache_management_object( async def _cache_team_object( team_id: str, team_table: LiteLLM_TeamTableCachedObj, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: Optional[ProxyLogging], ): key = "team_id:{}".format(team_id) @@ -1574,13 +1601,14 @@ async def _cache_team_object( value=team_table, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, + model_type=LiteLLM_TeamTableCachedObj, ) async def _cache_key_object( hashed_token: str, user_api_key_obj: UserAPIKeyAuth, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: Optional[ProxyLogging], ): key = hashed_token @@ -1593,12 +1621,13 @@ async def _cache_key_object( value=user_api_key_obj, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, + model_type=UserAPIKeyAuth, ) async def _delete_cache_key_object( hashed_token: str, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: Optional[ProxyLogging], ): key = hashed_token @@ -1646,7 +1675,7 @@ async def _get_team_object_from_db(team_id: str, prisma_client: PrismaClient): async def _get_team_object_from_user_api_key_cache( team_id: str, prisma_client: PrismaClient, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, last_db_access_time: LimitedSizeOrderedDict, db_cache_expiry: int, proxy_logging_obj: Optional[ProxyLogging], @@ -1707,38 +1736,38 @@ async def _get_team_object_from_user_api_key_cache( async def _get_team_object_from_cache( key: str, proxy_logging_obj: Optional[ProxyLogging], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span], ) -> Optional[LiteLLM_TeamTableCachedObj]: - cached_team_obj: Optional[LiteLLM_TeamTableCachedObj] = None - - ## CHECK REDIS CACHE ## + ## INTERNAL USAGE CACHE (plain DualCache) — checked before UserApiKeyCache stores ## if ( proxy_logging_obj is not None and proxy_logging_obj.internal_usage_cache.dual_cache ): - cached_team_obj = ( + cached_raw = ( await proxy_logging_obj.internal_usage_cache.dual_cache.async_get_cache( key=key, parent_otel_span=parent_otel_span ) ) + if cached_raw is not None: + from_internal = CacheCodec.deserialize( + cached_raw, LiteLLM_TeamTableCachedObj + ) + if from_internal is not None: + return from_internal - if cached_team_obj is None: - cached_team_obj = await user_api_key_cache.async_get_cache(key=key) - - if cached_team_obj is not None: - if isinstance(cached_team_obj, dict): - return LiteLLM_TeamTableCachedObj(**cached_team_obj) - elif isinstance(cached_team_obj, LiteLLM_TeamTableCachedObj): - return cached_team_obj - - return None + decoded = await user_api_key_cache.async_get_cache( + key=key, + parent_otel_span=parent_otel_span, + model_type=LiteLLM_TeamTableCachedObj, + ) + return decoded async def get_team_object( team_id: str, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, check_cache_only: Optional[bool] = None, @@ -1804,20 +1833,21 @@ async def get_team_object( async def _cache_access_object( access_group_id: str, access_group_table: LiteLLM_AccessGroupTable, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: Optional[ProxyLogging] = None, ): key = "access_group_id:{}".format(access_group_id) await user_api_key_cache.async_set_cache( key=key, value=access_group_table, + model_type=LiteLLM_AccessGroupTable, ttl=DEFAULT_ACCESS_GROUP_CACHE_TTL, ) async def _delete_cache_access_object( access_group_id: str, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: Optional[ProxyLogging] = None, ): key = "access_group_id:{}".format(access_group_id) @@ -1835,7 +1865,7 @@ async def _delete_cache_access_object( async def get_access_object( access_group_id: str, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> LiteLLM_AccessGroupTable: """ @@ -1857,13 +1887,12 @@ async def get_access_object( key = "access_group_id:{}".format(access_group_id) - # Always check cache first - cached_access_obj = await user_api_key_cache.async_get_cache(key=key) + cached_access_obj = await user_api_key_cache.async_get_cache( + key=key, + model_type=LiteLLM_AccessGroupTable, + ) if cached_access_obj is not None: - if isinstance(cached_access_obj, dict): - return LiteLLM_AccessGroupTable(**cached_access_obj) - elif isinstance(cached_access_obj, LiteLLM_AccessGroupTable): - return cached_access_obj + return cached_access_obj # Not in cache - fetch from DB try: @@ -1909,7 +1938,7 @@ async def get_access_object( async def get_team_object_by_alias( team_alias: str, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional["Span"] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> LiteLLM_TeamTableCachedObj: @@ -1991,6 +2020,7 @@ async def get_team_object_by_alias( await user_api_key_cache.async_set_cache( key=cache_key, value=team_obj, + model_type=LiteLLM_TeamTableCachedObj, ttl=DEFAULT_IN_MEMORY_TTL, ) # Also cache by team_id for consistency @@ -1998,6 +2028,7 @@ async def get_team_object_by_alias( await user_api_key_cache.async_set_cache( key=team_id_cache_key, value=team_obj, + model_type=LiteLLM_TeamTableCachedObj, ttl=DEFAULT_IN_MEMORY_TTL, ) @@ -2019,7 +2050,7 @@ async def get_team_object_by_alias( async def get_org_object_by_alias( org_alias: str, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional["Span"] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> Optional[LiteLLM_OrganizationTable]: @@ -2046,12 +2077,12 @@ async def get_org_object_by_alias( # Check cache first (keyed by alias) cache_key = "org_alias:{}".format(org_alias) - cached_org_obj = await user_api_key_cache.async_get_cache(key=cache_key) + cached_org_obj = await user_api_key_cache.async_get_cache( + key=cache_key, + model_type=LiteLLM_OrganizationTable, + ) if cached_org_obj is not None: - if isinstance(cached_org_obj, dict): - return LiteLLM_OrganizationTable(**cached_org_obj) - elif isinstance(cached_org_obj, LiteLLM_OrganizationTable): - return cached_org_obj + return cached_org_obj # Query database by organization_alias try: @@ -2081,13 +2112,15 @@ async def get_org_object_by_alias( # Cache the result await user_api_key_cache.async_set_cache( key=cache_key, - value=org_obj.model_dump(), + value=org_obj, + model_type=LiteLLM_OrganizationTable, ttl=DEFAULT_IN_MEMORY_TTL, ) # Also cache by org_id for consistency await user_api_key_cache.async_set_cache( key="org_id:{}".format(org_obj.organization_id), - value=org_obj.model_dump(), + value=org_obj, + model_type=LiteLLM_OrganizationTable, ttl=DEFAULT_IN_MEMORY_TTL, ) @@ -2290,7 +2323,7 @@ async def get_jwt_key_mapping_object( async def get_key_object( hashed_token: str, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, check_cache_only: Optional[bool] = None, @@ -2308,15 +2341,14 @@ async def get_key_object( # check if in cache key = hashed_token - cached_key_obj: Optional[UserAPIKeyAuth] = await user_api_key_cache.async_get_cache( - key=key + # Same flow as before: use cache only when we have a hit we can turn into UserAPIKeyAuth + # (dict from Redis / model_dump, or UserAPIKeyAuth from in-memory). Otherwise fall through to DB. + user_api_key_auth = await user_api_key_cache.async_get_cache( + key=key, + model_type=UserAPIKeyAuth, ) - - if cached_key_obj is not None: - if isinstance(cached_key_obj, dict): - return UserAPIKeyAuth(**cached_key_obj) - elif isinstance(cached_key_obj, UserAPIKeyAuth): - return cached_key_obj + if user_api_key_auth is not None: + return user_api_key_auth if check_cache_only: raise Exception( @@ -2373,7 +2405,7 @@ async def get_key_object( async def get_object_permission( object_permission_id: str, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> Optional[LiteLLM_ObjectPermissionTable]: @@ -2389,12 +2421,12 @@ async def get_object_permission( # check if in cache key = "object_permission_id:{}".format(object_permission_id) - cached_obj_permission = await user_api_key_cache.async_get_cache(key=key) - if cached_obj_permission is not None: - if isinstance(cached_obj_permission, dict): - return LiteLLM_ObjectPermissionTable(**cached_obj_permission) - elif isinstance(cached_obj_permission, LiteLLM_ObjectPermissionTable): - return cached_obj_permission + deserialized_perm = await user_api_key_cache.async_get_cache( + key=key, + model_type=LiteLLM_ObjectPermissionTable, + ) + if deserialized_perm is not None: + return deserialized_perm # else, check db try: @@ -2405,14 +2437,15 @@ async def get_object_permission( if response is None: return None - # save the object permission to cache + _perm_obj = LiteLLM_ObjectPermissionTable(**response.dict()) await user_api_key_cache.async_set_cache( key=key, - value=response.model_dump(), + value=_perm_obj, + model_type=LiteLLM_ObjectPermissionTable, ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, ) - return LiteLLM_ObjectPermissionTable(**response.dict()) + return _perm_obj except Exception: return None @@ -2421,7 +2454,7 @@ async def get_object_permission( async def get_managed_vector_store_rows_by_uuids( uuids: List[str], prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> List[LiteLLM_ManagedVectorStoresTable]: @@ -2441,14 +2474,12 @@ async def get_managed_vector_store_rows_by_uuids( for uuid in uuids: key = "managed_vector_store_id:{}".format(uuid) - cached = await user_api_key_cache.async_get_cache(key=key) - if cached is not None: - if isinstance(cached, dict): - result.append(LiteLLM_ManagedVectorStoresTable(**cached)) - elif isinstance(cached, LiteLLM_ManagedVectorStoresTable): - result.append(cached) - else: - cache_misses.append(uuid) + deserialized_vs = await user_api_key_cache.async_get_cache( + key=key, + model_type=LiteLLM_ManagedVectorStoresTable, + ) + if deserialized_vs is not None: + result.append(deserialized_vs) else: cache_misses.append(uuid) @@ -2474,7 +2505,8 @@ async def get_managed_vector_store_rows_by_uuids( key = "managed_vector_store_id:{}".format(cached_obj.vector_store_id) await user_api_key_cache.async_set_cache( key=key, - value=row_dict, + value=cached_obj, + model_type=LiteLLM_ManagedVectorStoresTable, ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, ) result.append(cached_obj) @@ -2486,7 +2518,7 @@ async def get_managed_vector_store_rows_by_uuids( async def get_org_object( org_id: str, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, include_budget_table: bool = False, @@ -2517,12 +2549,12 @@ async def get_org_object( cache_key = "org_id:{}:with_budget".format(org_id) # check if in cache - cached_org_obj = user_api_key_cache.async_get_cache(key=cache_key) - if cached_org_obj is not None: - if isinstance(cached_org_obj, dict): - return LiteLLM_OrganizationTable(**cached_org_obj) - elif isinstance(cached_org_obj, LiteLLM_OrganizationTable): - return cached_org_obj + deserialized_org = await user_api_key_cache.async_get_cache( + key=cache_key, + model_type=LiteLLM_OrganizationTable, + ) + if deserialized_org is not None: + return deserialized_org # else, check db try: query_kwargs: Dict[str, Any] = {"where": {"organization_id": org_id}} @@ -2536,16 +2568,16 @@ async def get_org_object( if response is None: raise Exception + _org_obj = LiteLLM_OrganizationTable(**response.model_dump()) # Cache the result await user_api_key_cache.async_set_cache( key=cache_key, - value=( - response.model_dump() if hasattr(response, "model_dump") else response - ), + value=_org_obj, + model_type=LiteLLM_OrganizationTable, ttl=DEFAULT_IN_MEMORY_TTL, ) - return response + return _org_obj except Exception: raise Exception( f"Organization doesn't exist in db. Organization={org_id}. Create organization via `/organization/new` call." @@ -2558,7 +2590,7 @@ async def _get_resources_from_access_groups( "access_model_names", "access_mcp_server_ids", "access_agent_ids" ], prisma_client: Optional[PrismaClient] = None, - user_api_key_cache: Optional[DualCache] = None, + user_api_key_cache: Optional[UserApiKeyCache] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> List[str]: """ @@ -2616,7 +2648,7 @@ async def _get_resources_from_access_groups( async def _get_models_from_access_groups( access_group_ids: List[str], prisma_client: Optional[PrismaClient] = None, - user_api_key_cache: Optional[DualCache] = None, + user_api_key_cache: Optional[UserApiKeyCache] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> List[str]: """ @@ -2635,7 +2667,7 @@ async def _get_models_from_access_groups( async def _get_mcp_server_ids_from_access_groups( access_group_ids: List[str], prisma_client: Optional[PrismaClient] = None, - user_api_key_cache: Optional[DualCache] = None, + user_api_key_cache: Optional[UserApiKeyCache] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> List[str]: """ @@ -2654,7 +2686,7 @@ async def _get_mcp_server_ids_from_access_groups( async def _get_agent_ids_from_access_groups( access_group_ids: List[str], prisma_client: Optional[PrismaClient] = None, - user_api_key_cache: Optional[DualCache] = None, + user_api_key_cache: Optional[UserApiKeyCache] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> List[str]: """ @@ -2962,6 +2994,116 @@ async def can_user_call_model( ) +def _search_tool_names_from_object_permission( + object_permission: Optional[LiteLLM_ObjectPermissionTable], +) -> List[str]: + """Return allowlisted search tool names from object_permission (empty = unrestricted).""" + if object_permission is None: + return [] + raw = object_permission.search_tools + if not raw: + return [] + return list(raw) + + +def _can_object_call_search_tool( + search_tool_name: str, + allowed_search_tools: List[str], + object_type: Literal["key", "team", "project"], +) -> Literal[True]: + """ + Check if an object (key/team/project) can access a specific search tool. + + Similar to _can_object_call_model but for search tools. + + Args: + search_tool_name: The search tool being requested + allowed_search_tools: List of allowed search tool names for this object + object_type: Type of object for error messaging + + Returns: + True if access is allowed + + Raises: + ProxyException if access is denied + """ + # Empty list means all search tools are allowed + if not allowed_search_tools: + return True + + # Check if the search tool is in the allowlist + if search_tool_name in allowed_search_tools: + return True + + # Access denied + raise ProxyException( + message=f"{object_type.capitalize()} not allowed to access search tool: {search_tool_name}. " + f"Allowed search tools: {allowed_search_tools}", + type=ProxyErrorTypes.key_model_access_denied, + param="search_tool_name", + code=status.HTTP_403_FORBIDDEN, + ) + + +async def can_key_call_search_tool( + search_tool_name: str, + valid_token: UserAPIKeyAuth, +) -> Literal[True]: + """ + Check if a key can access a specific search tool. + + Similar to can_key_call_model but for search tools. + + Args: + search_tool_name: The search tool being requested + valid_token: The authenticated key + + Returns: + True if access is allowed + + Raises: + ProxyException if access is denied + """ + return _can_object_call_search_tool( + search_tool_name=search_tool_name, + allowed_search_tools=_search_tool_names_from_object_permission( + valid_token.object_permission + ), + object_type="key", + ) + + +async def can_team_call_search_tool( + search_tool_name: str, + team_object: Optional[LiteLLM_TeamTable], +) -> Literal[True]: + """ + Check if a team can access a specific search tool. + + Similar to can_team_access_model but for search tools. + + Args: + search_tool_name: The search tool being requested + team_object: The team object + + Returns: + True if access is allowed + + Raises: + ProxyException if access is denied + """ + if team_object is None: + return True + + return _can_object_call_search_tool( + search_tool_name=search_tool_name, + allowed_search_tools=_search_tool_names_from_object_permission( + team_object.object_permission + ), + object_type="team", + ) + + async def is_valid_fallback_model( model: str, llm_router: Optional[Router], @@ -3268,7 +3410,7 @@ async def _check_team_member_budget( user_object: Optional[LiteLLM_UserTable], valid_token: Optional[UserAPIKeyAuth], prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, ): """Check if team member is over their max budget within the team.""" @@ -3293,6 +3435,7 @@ async def _check_team_member_budget( if ( team_membership is not None and team_membership.litellm_budget_table is not None + and team_membership.litellm_budget_table.max_budget is not None ): team_member_budget = team_membership.litellm_budget_table.max_budget else: @@ -3335,7 +3478,7 @@ async def _check_team_member_model_access( valid_token: UserAPIKeyAuth, llm_router: Optional[Router], prisma_client: Optional["PrismaClient"], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, ) -> None: """ @@ -3642,7 +3785,7 @@ async def _project_soft_budget_check( async def get_project_object( project_id: str, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> Optional[LiteLLM_ProjectTableCachedObj]: """ @@ -3657,12 +3800,12 @@ async def get_project_object( # Check cache first cache_key = "project_id:{}".format(project_id) - cached_obj = await user_api_key_cache.async_get_cache(key=cache_key) - if cached_obj is not None: - if isinstance(cached_obj, dict): - return LiteLLM_ProjectTableCachedObj(**cached_obj) - elif isinstance(cached_obj, LiteLLM_ProjectTableCachedObj): - return cached_obj + deserialized_project = await user_api_key_cache.async_get_cache( + key=cache_key, + model_type=LiteLLM_ProjectTableCachedObj, + ) + if deserialized_project is not None: + return deserialized_project # Fetch from DB project_row = await prisma_client.db.litellm_projecttable.find_unique( @@ -3681,6 +3824,7 @@ async def get_project_object( value=project_obj, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, + model_type=LiteLLM_ProjectTableCachedObj, ) return project_obj @@ -3690,7 +3834,7 @@ async def _organization_max_budget_check( valid_token: Optional[UserAPIKeyAuth], team_object: Optional[LiteLLM_TeamTable], prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, ): """ @@ -3784,7 +3928,7 @@ async def _organization_max_budget_check( async def _tag_max_budget_check( request_body: dict, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, valid_token: Optional[UserAPIKeyAuth], ): diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 91c8f2dd7c9..cbed34adacf 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -167,6 +167,81 @@ def _allow_model_level_clientside_configurable_parameters( ) +# Config dicts whose entries are spread as ``**dict`` into outbound LLM +# API calls. ``litellm_embedding_config`` is consumed by the Milvus +# vector store transformer; future nested-config keys with the same +# threat shape should be added here. +_NESTED_CONFIG_KEYS: Tuple[str, ...] = ("litellm_embedding_config",) + +# Banned root-level params. Same list applies to every entry in +# ``_NESTED_CONFIG_KEYS`` because those dicts get spread as ``**kwargs`` +# into the same outbound calls. +_BANNED_REQUEST_BODY_PARAMS: Tuple[str, ...] = ( + "api_base", + "base_url", + "user_config", + "aws_sts_endpoint", + "aws_web_identity_token", + "aws_role_name", + "vertex_credentials", + # Endpoint-targeting fields that retarget the outbound request or + # an observability callback. An attacker-controlled value either + # exfiltrates the request payload (incl. messages + admin-set + # tokens) to the attacker's host, or coerces the proxy into + # authenticating against the attacker's host with admin secrets. + "aws_bedrock_runtime_endpoint", + "langsmith_base_url", + "langfuse_host", + "posthog_host", + "braintrust_host", + "slack_webhook_url", + # Provider-specific endpoint overrides that flow into the outbound + # request via ``optional_params``. Same threat as ``api_base``: + # ``s3_endpoint_url`` redirects Bedrock file uploads to attacker + # S3; ``sagemaker_base_url`` redirects all SageMaker traffic; + # ``deployment_url`` redirects SAP deployments. + "s3_endpoint_url", + "sagemaker_base_url", + "deployment_url", +) + + +def _check_banned_params( + body: dict, + general_settings: dict, + llm_router: Optional[Router], + model: str, +) -> None: + """Raise ``ValueError`` if ``body`` carries a banned param without admin opt-in. + + Shared between the root-level check and the nested-config check so a + new banned param only needs to be added in one place. + """ + for param in _BANNED_REQUEST_BODY_PARAMS: + if param not in body: + continue + if general_settings.get("allow_client_side_credentials") is True: + return + if ( + _allow_model_level_clientside_configurable_parameters( + model=model, + param=param, + request_body_value=body[param], + llm_router=llm_router, + ) + is True + ): + return + raise ValueError( + f"Rejected Request: {param} is not allowed in request body. " + "Clientside passthrough requires explicit admin opt-in via " + "either `general_settings.allow_client_side_credentials = true` " + "(proxy-wide) or `configurable_clientside_auth_params` on the " + "deployment in your proxy config.yaml. " + "Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997", + ) + + def is_request_body_safe( request_body: dict, general_settings: dict, llm_router: Optional[Router], model: str ) -> bool: @@ -175,72 +250,31 @@ def is_request_body_safe( A malicious user can set the api_base to their own domain and invoke POST /chat/completions to intercept and steal the OpenAI API key. Relevant issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997 + + The blocklist is enforced unconditionally. Legitimate clientside + credential / endpoint passthrough goes through one of the two + explicit admin opt-ins (``general_settings.allow_client_side_credentials`` + proxy-wide or ``configurable_clientside_auth_params`` per deployment). + Historically there was a third, *implicit*, *caller-controlled* path: + ``check_complete_credentials`` returned True when the caller supplied + any non-empty ``api_key``, which made the entire blocklist a no-op. + That bypass turned every missing entry on the blocklist into an + exploitable SSRF / credential-exfil hole — see GHSA-jh89-88fc-qrfp, + GHSA-3frq-6r6h-7j64, and the chain of veria-admin findings (Dv_m860l, + b_yRJeQ5, stN90yjP, LBlyOAc8, U2TD78kg). Removed: the blocklist now + has a single, predictable failure mode for missing entries (a 400), + not a credential leak. + + Iterative single-level descent into ``_NESTED_CONFIG_KEYS`` (rather + than recursion) covers nested-config attacks like Milvus's + ``litellm_embedding_config.api_base`` (VERIA-6) without exposing a + recursion-depth DoS surface. """ - banned_params = [ - "api_base", - "base_url", - "user_config", - "aws_sts_endpoint", - "aws_web_identity_token", - "aws_role_name", - "vertex_credentials", - # Endpoint-targeting fields that retarget the outbound request or - # an observability callback. An attacker-controlled value either - # exfiltrates the request payload (incl. messages + admin-set - # tokens) to the attacker's host, or coerces the proxy into - # authenticating against the attacker's host with admin secrets. - "aws_bedrock_runtime_endpoint", - "langsmith_base_url", - "langfuse_host", - "posthog_host", - "braintrust_host", - "slack_webhook_url", - # Provider-specific endpoint overrides that flow into the outbound - # request via ``optional_params``. Same threat as ``api_base``: - # ``s3_endpoint_url`` redirects Bedrock file uploads to attacker - # S3; ``sagemaker_base_url`` redirects all SageMaker traffic; - # ``deployment_url`` redirects SAP deployments. - "s3_endpoint_url", - "sagemaker_base_url", - "deployment_url", - ] - - # The blocklist is enforced unconditionally. Legitimate clientside - # credential / endpoint passthrough goes through one of the two - # explicit admin opt-ins (``general_settings.allow_client_side_credentials`` - # proxy-wide or ``configurable_clientside_auth_params`` per deployment). - # Historically there was a third, *implicit*, *caller-controlled* path: - # ``check_complete_credentials`` returned True when the caller supplied - # any non-empty ``api_key``, which made the entire blocklist a no-op. - # That bypass turned every missing entry on the blocklist into an - # exploitable SSRF / credential-exfil hole — see GHSA-jh89-88fc-qrfp, - # GHSA-3frq-6r6h-7j64, and the chain of veria-admin findings (Dv_m860l, - # b_yRJeQ5, stN90yjP, LBlyOAc8, U2TD78kg). Removed: the blocklist now - # has a single, predictable failure mode for missing entries (a 400), - # not a credential leak. - for param in banned_params: - if param in request_body: - if general_settings.get("allow_client_side_credentials") is True: - return True - elif ( - _allow_model_level_clientside_configurable_parameters( - model=model, - param=param, - request_body_value=request_body[param], - llm_router=llm_router, - ) - is True - ): - return True - raise ValueError( - f"Rejected Request: {param} is not allowed in request body. " - "Clientside passthrough requires explicit admin opt-in via " - "either `general_settings.allow_client_side_credentials = true` " - "(proxy-wide) or `configurable_clientside_auth_params` on the " - "deployment in your proxy config.yaml. " - "Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997", - ) - + _check_banned_params(request_body, general_settings, llm_router, model) + for nested_key in _NESTED_CONFIG_KEYS: + nested = request_body.get(nested_key) + if isinstance(nested, dict): + _check_banned_params(nested, general_settings, llm_router, model) return True diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index f50c950d747..71411bed7fd 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -6,6 +6,8 @@ Currently only supports admin. JWT token must have 'litellm_proxy_admin' in scope. """ +from __future__ import annotations + import fnmatch import hashlib import os @@ -20,7 +22,6 @@ import jwt from jwt.api_jwk import PyJWK from litellm._logging import verbose_proxy_logger -from litellm.caching.caching import DualCache from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value from litellm.llms.custom_httpx.httpx_handler import HTTPHandler @@ -46,6 +47,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_checks import can_team_access_model from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.utils import PrismaClient, ProxyLogging from .auth_checks import ( @@ -73,7 +75,7 @@ class JWTHandler: """ prisma_client: Optional[PrismaClient] - user_api_key_cache: DualCache + user_api_key_cache: UserApiKeyCache # Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html # "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret # the key in different ways (e.g. HS* and RS*)." @@ -99,7 +101,7 @@ class JWTHandler: def update_environment( self, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, litellm_jwtauth: LiteLLM_JWTAuth, leeway: int = 0, ) -> None: @@ -952,7 +954,7 @@ class JWTAuthManager: jwt_handler: JWTHandler, jwt_valid_token: dict, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span], proxy_logging_obj: ProxyLogging, ) -> Tuple[Optional[str], Optional[LiteLLM_TeamTable]]: @@ -1045,7 +1047,7 @@ class JWTAuthManager: route: str, jwt_handler: JWTHandler, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span], proxy_logging_obj: ProxyLogging, ) -> Tuple[Optional[str], Optional[LiteLLM_TeamTable]]: @@ -1133,7 +1135,7 @@ class JWTAuthManager: valid_user_email: Optional[bool], jwt_handler: JWTHandler, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span], proxy_logging_obj: ProxyLogging, route: str, @@ -1349,7 +1351,7 @@ class JWTAuthManager: jwt_valid_token: dict, user_object: Optional[LiteLLM_UserTable], prisma_client: Optional[PrismaClient], - user_api_key_cache: Optional[DualCache] = None, + user_api_key_cache: Optional[UserApiKeyCache] = None, ) -> None: """ Sync user role and team memberships with JWT claims @@ -1377,7 +1379,8 @@ class JWTAuthManager: if user_api_key_cache is not None: await user_api_key_cache.async_set_cache( key=user_object.user_id, - value=user_object.model_dump(), + value=user_object, + model_type=LiteLLM_UserTable, ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, ) @@ -1400,7 +1403,8 @@ class JWTAuthManager: if user_api_key_cache is not None: await user_api_key_cache.async_set_cache( key=user_object.user_id, - value=user_object.model_dump(), + value=user_object, + model_type=LiteLLM_UserTable, ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, ) return None @@ -1412,7 +1416,7 @@ class JWTAuthManager: request_headers: Optional[dict], jwt_handler: JWTHandler, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span], proxy_logging_obj: ProxyLogging, ) -> None: @@ -1456,7 +1460,7 @@ class JWTAuthManager: user_object: Optional[LiteLLM_UserTable], user_id: Optional[str], prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span], proxy_logging_obj: ProxyLogging, team_id_upsert: Optional[bool], @@ -1514,7 +1518,7 @@ class JWTAuthManager: general_settings: dict, route: str, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span], proxy_logging_obj: ProxyLogging, request_headers: Optional[dict] = None, diff --git a/litellm/proxy/auth/ip_address_utils.py b/litellm/proxy/auth/ip_address_utils.py index 34fab4849e5..39d3282942f 100644 --- a/litellm/proxy/auth/ip_address_utils.py +++ b/litellm/proxy/auth/ip_address_utils.py @@ -13,6 +13,10 @@ from fastapi import Request from litellm._logging import verbose_proxy_logger from litellm.proxy.auth.auth_utils import _get_request_ip_address +# One-shot warning so operators upgrading from the prior "always trust X-Forwarded-*" +# behaviour see an actionable message in their logs the first time it triggers. +_warned_xff_without_trusted_ranges = False + class IPAddressUtils: """Static utilities for IP-based MCP access control.""" @@ -106,6 +110,61 @@ class IPAddressUtils: return any(addr in network for network in networks) + @staticmethod + def is_request_from_trusted_proxy( + request: Request, + general_settings: Optional[Dict[str, Any]] = None, + ) -> bool: + """ + Return True if X-Forwarded-* headers on this request should be trusted. + + Trusts the headers iff both: + 1. ``use_x_forwarded_for`` is enabled in proxy settings, AND + 2. ``mcp_trusted_proxy_ranges`` is configured AND the direct + connection IP (``request.client.host``) falls inside one of + those CIDRs. + + When ``use_x_forwarded_for`` is enabled but ``mcp_trusted_proxy_ranges`` + is missing, the headers are NOT trusted: there is no way to + distinguish a trusted reverse proxy from a direct attacker, so callers + that build URLs (OAuth issuer / redirect_uri / etc.) must fall back + to the request's literal base URL instead of risking a poisoned host. + """ + if general_settings is None: + try: + from litellm.proxy.proxy_server import ( + general_settings as proxy_general_settings, + ) + + general_settings = proxy_general_settings + except ImportError: + general_settings = {} + + if general_settings is None: + general_settings = {} + + if not general_settings.get("use_x_forwarded_for", False): + return False + + trusted_ranges = general_settings.get("mcp_trusted_proxy_ranges") + if not trusted_ranges: + global _warned_xff_without_trusted_ranges + if not _warned_xff_without_trusted_ranges: + verbose_proxy_logger.warning( + "use_x_forwarded_for is enabled but mcp_trusted_proxy_ranges " + "is not configured. X-Forwarded-* headers will NOT be " + "trusted, so MCP OAuth discovery URLs will use the proxy's " + "literal base URL. Set mcp_trusted_proxy_ranges in " + "general_settings to your reverse-proxy CIDR(s) to allow " + "X-Forwarded-* through." + ) + _warned_xff_without_trusted_ranges = True + return False + + direct_ip = request.client.host if request.client else None + trusted_networks = IPAddressUtils.parse_trusted_proxy_networks(trusted_ranges) + return IPAddressUtils.is_trusted_proxy(direct_ip, trusted_networks) + @staticmethod def get_mcp_client_ip( request: Request, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index b8db3cd2a7b..bfd1f2e0b3a 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -20,7 +20,7 @@ from fastapi.security.api_key import APIKeyHeader import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging -from litellm.caching import DualCache +from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value from litellm.proxy._types import * @@ -59,6 +59,7 @@ from litellm.proxy.auth.oauth2_check import Oauth2Handler from litellm.proxy.auth.oauth2_proxy_hook import handle_oauth2_proxy_request from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.cache_coordinator import EventDrivenCacheCoordinator +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, _safe_get_request_headers, @@ -328,7 +329,7 @@ _global_spend_coordinator = EventDrivenCacheCoordinator(log_prefix="[GLOBAL SPEN async def _fetch_global_spend_with_event_coordination( cache_key: str, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, prisma_client: PrismaClient, ) -> Optional[float]: """ @@ -344,14 +345,14 @@ async def _fetch_global_spend_with_event_coordination( return await _global_spend_coordinator.get_or_load( cache_key=cache_key, - cache=user_api_key_cache, + cache=user_api_key_cache, # pyright: ignore[reportArgumentType] load_fn=_load_global_spend, ) async def get_global_proxy_spend( litellm_proxy_admin_name: str, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, prisma_client: Optional[PrismaClient], token: str, proxy_logging_obj: ProxyLogging, @@ -472,7 +473,12 @@ async def check_api_key_for_custom_headers_or_pass_through_endpoints( for endpoint in pass_through_endpoints: if isinstance(endpoint, dict) and endpoint.get("path", "") == route: ## IF AUTH DISABLED - if endpoint.get("auth") is not True: + # Default to True: a config dict with no ``auth`` key + # otherwise produced an unauthenticated forwarder. The + # Pydantic ``PassThroughGenericEndpoint.auth`` default + # is also True, but raw config dicts skip that path — + # so this runtime check has to default to True too. + if endpoint.get("auth", True) is not True: return UserAPIKeyAuth() ## IF AUTH ENABLED ### IF CUSTOM PARSER REQUIRED @@ -504,7 +510,7 @@ async def _resolve_jwt_to_virtual_key( jwt_claims: dict, jwt_handler: JWTHandler, prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, parent_otel_span: Optional[Span], proxy_logging_obj: ProxyLogging, ) -> Optional[UserAPIKeyAuth]: @@ -1106,9 +1112,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 is_master_key_valid = False ## VALIDATE MASTER KEY ## - try: - assert isinstance(master_key, str) - except Exception: + if not isinstance(master_key, str): raise HTTPException( status_code=500, detail={ @@ -1119,10 +1123,14 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ) if is_master_key_valid: + # Substitute a stable alias for the raw master key so neither the + # master key nor its hash propagates into spend logs, Prometheus + # /metrics labels, audit trails, rate-limit buckets, or any other + # downstream consumer of UserAPIKeyAuth.api_key. _user_api_key_obj = await _return_user_api_key_auth_obj( user_obj=None, user_role=LitellmUserRoles.PROXY_ADMIN, - api_key=master_key, + api_key=LITELLM_PROXY_MASTER_KEY_ALIAS, parent_otel_span=parent_otel_span, valid_token_dict={ **end_user_params, @@ -1174,11 +1182,15 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 if len(api_key) > 8 else "****" ) - assert api_key.startswith( - "sk-" - ), "LiteLLM Virtual Key expected. Received={}, expected to start with 'sk-'.".format( - _masked_key - ) # prevent token hashes from being used + if not api_key.startswith("sk-"): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=( + "LiteLLM Virtual Key expected. Received={}, expected to start with 'sk-'.".format( + _masked_key + ) + ), + ) # prevent token hashes from being used else: verbose_logger.warning( "litellm.proxy.proxy_server.user_api_key_auth(): Warning - Key is not a string. Got type={}".format( @@ -1286,7 +1298,8 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 _cache_key = f"{valid_token.team_id}_{valid_token.user_id}" team_member_info = await user_api_key_cache.async_get_cache( - key=_cache_key + key=_cache_key, + model_type=LiteLLM_TeamMembership, ) if team_member_info is None: # read from DB @@ -1294,18 +1307,23 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 _team_id = valid_token.team_id if _user_id is not None and _team_id is not None: - team_member_info = await prisma_client.db.litellm_teammembership.find_first( + _db_member = await prisma_client.db.litellm_teammembership.find_first( where={ "user_id": _user_id, "team_id": _team_id, }, # type: ignore include={"litellm_budget_table": True}, ) - await user_api_key_cache.async_set_cache( - key=_cache_key, - value=team_member_info, - ttl=5, - ) + if _db_member is not None: + team_member_info = LiteLLM_TeamMembership( + **_db_member.dict() + ) + await user_api_key_cache.async_set_cache( + key=_cache_key, + value=team_member_info, + model_type=LiteLLM_TeamMembership, + ttl=5, + ) if ( team_member_info is not None @@ -1452,9 +1470,13 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 else: valid_token.team_object_permission = None - await user_api_key_cache.async_set_cache( - key=valid_token.team_id, value=_team_obj - ) # save team table in cache - used for tpm/rpm limiting - tpm_rpm_limiter.py + # Only cache when the key is a real team_id (non-team keys must not use key=None). + if valid_token.team_id is not None and _team_obj is not None: + await user_api_key_cache.async_set_cache( + key=valid_token.team_id, + value=_team_obj, + model_type=LiteLLM_TeamTableCachedObj, + ) # save team table in cache - used for tpm/rpm limiting - tpm_rpm_limiter.py # Fetch project object if key belongs to a project _project_obj = None diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 160e9c23f01..935b96a0e39 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -474,6 +474,10 @@ async def retrieve_batch( # noqa: PLR0915 ) # Fix: The helper sets "file_id" but we need "batch_id" data["batch_id"] = data.pop("file_id", original_batch_id) + # Provider-config providers (e.g. bedrock) require `model` in kwargs + # so litellm.aretrieve_batch can load BedrockBatchesConfig. Without + # it the call falls into the legacy provider switch and 400s. + data["model"] = model_from_id # Retrieve batch using model credentials response = await litellm.aretrieve_batch( diff --git a/litellm/proxy/client/README.md b/litellm/proxy/client/README.md index 5dcc88cacbe..adf562d69c5 100644 --- a/litellm/proxy/client/README.md +++ b/litellm/proxy/client/README.md @@ -313,23 +313,24 @@ sequenceDiagram participant Proxy as LiteLLM Proxy participant SSO as SSO Provider - CLI->>CLI: Generate key ID (sk-uuid) - CLI->>Browser: Open /sso/key/generate?source=litellm-cli&key=sk-uuid + CLI->>Proxy: POST /sso/cli/start + Proxy->>CLI: Return login_id, poll_secret, user_code + CLI->>Browser: Open /sso/key/generate?source=litellm-cli&key=login_id - Browser->>Proxy: GET /sso/key/generate?source=litellm-cli&key=sk-uuid - Proxy->>Proxy: Set cli_state = litellm-session-token:sk-uuid - Proxy->>SSO: Redirect with state=litellm-session-token:sk-uuid + Browser->>Proxy: GET /sso/key/generate?source=litellm-cli&key=login_id + Proxy->>Proxy: Set cli_state = litellm-session-token:login_id + Proxy->>SSO: Redirect with state=litellm-session-token:login_id SSO->>Browser: Show login page Browser->>SSO: User authenticates - SSO->>Proxy: Redirect to /sso/callback?state=litellm-session-token:sk-uuid + SSO->>Proxy: Redirect to /sso/callback?state=litellm-session-token:login_id Proxy->>Proxy: Check if state starts with "litellm-session-token:" - Proxy->>Proxy: Generate API key with ID=sk-uuid - Proxy->>Browser: Show success page + Proxy->>Browser: Prompt for user_code + Browser->>Proxy: POST /sso/cli/complete/login_id - CLI->>Proxy: Poll /sso/cli/poll/sk-uuid - Proxy->>CLI: Return {"status": "ready", "key": "sk-uuid"} + CLI->>Proxy: Poll /sso/cli/poll/login_id with poll_secret header + Proxy->>CLI: Return {"status": "ready", "key": "jwt"} CLI->>CLI: Save key to ~/.litellm/token.json ``` @@ -343,13 +344,13 @@ The CLI provides three authentication commands: ### Authentication Flow Steps -1. **Generate Session ID**: CLI generates a unique key ID (`sk-{uuid}`) -2. **Open Browser**: CLI opens browser to `/sso/key/generate` with CLI source and key parameters -3. **SSO Redirect**: Proxy sets the formatted state (`litellm-session-token:sk-uuid`) as OAuth state parameter and redirects to SSO provider +1. **Start Session**: CLI creates a short-lived login session with `/sso/cli/start` +2. **Open Browser**: CLI opens browser to `/sso/key/generate` with CLI source and login ID parameters +3. **SSO Redirect**: Proxy sets the formatted state (`litellm-session-token:{login_id}`) as OAuth state parameter and redirects to SSO provider 4. **User Authentication**: User completes SSO authentication in browser 5. **Callback Processing**: SSO provider redirects back to proxy with state parameter -6. **Key Generation**: Proxy detects CLI login (state starts with "litellm-session-token:") and generates API key with pre-specified ID -7. **Polling**: CLI polls `/sso/cli/poll/{key_id}` endpoint until key is ready +6. **User Code Verification**: Browser confirms the verification code shown in the CLI +7. **Polling**: CLI polls `/sso/cli/poll/{login_id}` with the polling secret header until the JWT is ready 8. **Token Storage**: CLI saves the authentication token to `~/.litellm/token.json` ### Benefits of This Approach @@ -357,7 +358,7 @@ The CLI provides three authentication commands: - **No Local Server**: No need to run a local callback server - **Standard OAuth**: Uses OAuth 2.0 state parameter correctly - **Remote Compatible**: Works with remote proxy servers -- **Secure**: Uses UUID session identifiers +- **Secure**: Keeps the polling secret out of the browser handoff - **Simple Setup**: No additional OAuth redirect URL configuration needed ### Token Storage diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index aeb59e78a53..447837c35e7 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -5,6 +5,7 @@ import time import webbrowser from pathlib import Path from typing import Any, Dict, List, Optional +from urllib.parse import urlencode import click import requests @@ -52,12 +53,16 @@ def clear_token() -> None: os.remove(token_file) -def get_stored_api_key() -> Optional[str]: - """Get the stored API key from token file""" - # Use the SDK-level utility +def get_stored_api_key(expected_base_url: Optional[str] = None) -> Optional[str]: + """Get the stored API key from token file. + + If expected_base_url is provided, the key is only returned when it was + originally issued for that URL. This prevents credential leakage when the + CLI is pointed at a different (possibly malicious) server. + """ from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key - return get_litellm_gateway_api_key() + return get_litellm_gateway_api_key(expected_base_url=expected_base_url) # Team selection utilities @@ -241,7 +246,7 @@ def prompt_team_selection(teams: List[Dict[str, Any]]) -> Optional[Dict[str, Any def prompt_team_selection_fallback( - teams: List[Dict[str, Any]] + teams: List[Dict[str, Any]], ) -> Optional[Dict[str, Any]]: """Fallback team selection for non-interactive environments""" if not teams: @@ -279,6 +284,7 @@ def prompt_team_selection_fallback( def _poll_for_ready_data( url: str, *, + headers: Optional[Dict[str, str]] = None, total_timeout: int = 300, poll_interval: int = 2, request_timeout: int = 10, @@ -291,7 +297,10 @@ def _poll_for_ready_data( ) -> Optional[Dict[str, Any]]: for attempt in range(total_timeout // poll_interval): try: - response = requests.get(url, timeout=request_timeout) + request_kwargs: Dict[str, Any] = {"timeout": request_timeout} + if headers is not None: + request_kwargs["headers"] = headers + response = requests.get(url, **request_kwargs) if response.status_code == 200: data = response.json() status = data.get("status") @@ -346,7 +355,23 @@ def _normalize_teams(teams, team_details): return [] -def _poll_for_authentication(base_url: str, key_id: str) -> Optional[dict]: +def _start_cli_sso_flow(base_url: str) -> Dict[str, Any]: + response = requests.post(f"{base_url}/sso/cli/start", timeout=10) + response.raise_for_status() + data = response.json() + required_fields = ("login_id", "poll_secret", "user_code") + if not all(isinstance(data.get(field), str) for field in required_fields): + raise ValueError("Invalid CLI SSO start response") + return data + + +def _get_cli_sso_poll_headers(poll_secret: str) -> Dict[str, str]: + return {"x-litellm-cli-poll-secret": poll_secret} + + +def _poll_for_authentication( + base_url: str, key_id: str, poll_secret: str +) -> Optional[dict]: """ Poll the server for authentication completion and handle team selection. @@ -356,6 +381,7 @@ def _poll_for_authentication(base_url: str, key_id: str) -> Optional[dict]: poll_url = f"{base_url}/sso/cli/poll/{key_id}" data = _poll_for_ready_data( poll_url, + headers=_get_cli_sso_poll_headers(poll_secret), pending_message="Still waiting for authentication...", ) if not data: @@ -373,6 +399,7 @@ def _poll_for_authentication(base_url: str, key_id: str) -> Optional[dict]: jwt_with_team = _handle_team_selection_during_polling( base_url=base_url, key_id=key_id, + poll_secret=poll_secret, teams=normalized_teams, ) @@ -410,7 +437,7 @@ def _poll_for_authentication(base_url: str, key_id: str) -> Optional[dict]: def _handle_team_selection_during_polling( - base_url: str, key_id: str, teams: List[Dict[str, Any]] + base_url: str, key_id: str, poll_secret: str, teams: List[Dict[str, Any]] ) -> Optional[str]: """ Handle team selection and re-poll with selected team_id. @@ -441,6 +468,7 @@ def _handle_team_selection_during_polling( poll_url = f"{base_url}/sso/cli/poll/{key_id}?team_id={team_id}" data = _poll_for_ready_data( poll_url, + headers=_get_cli_sso_poll_headers(poll_secret), pending_message="Still waiting for team authentication...", other_status_message="Waiting for team authentication to complete...", http_error_log_every=10, @@ -514,29 +542,24 @@ def _render_and_prompt_for_team_selection(teams: List[Dict[str, Any]]) -> Option @click.pass_context def login(ctx: click.Context): """Login to LiteLLM proxy using SSO authentication""" - from litellm._uuid import uuid from litellm.constants import LITELLM_CLI_SOURCE_IDENTIFIER from litellm.proxy.client.cli.interface import show_commands base_url = ctx.obj["base_url"] - # Check if we have an existing key to regenerate - existing_key = get_stored_api_key() - - # Generate unique key ID for this login session - key_id = f"sk-{str(uuid.uuid4())}" - try: - # Construct SSO login URL with CLI source and pre-generated key - sso_url = f"{base_url}/sso/key/generate?source={LITELLM_CLI_SOURCE_IDENTIFIER}&key={key_id}" + cli_sso_flow = _start_cli_sso_flow(base_url=base_url) + key_id = cli_sso_flow["login_id"] + poll_secret = cli_sso_flow["poll_secret"] + user_code = cli_sso_flow["user_code"] - # If we have an existing key, include it as a parameter to the login endpoint - # The server will encode it in the OAuth state parameter for the SSO flow - if existing_key: - sso_url += f"&existing_key={existing_key}" + sso_url = f"{base_url}/sso/key/generate?" + urlencode( + {"source": LITELLM_CLI_SOURCE_IDENTIFIER, "key": key_id} + ) click.echo(f"Opening browser to: {sso_url}") click.echo("Please complete the SSO authentication in your browser...") + click.echo(f"Verification code: {user_code}") click.echo(f"Session ID: {key_id}") # Open browser @@ -545,15 +568,19 @@ def login(ctx: click.Context): # Poll for authentication completion click.echo("Waiting for authentication...") - auth_result = _poll_for_authentication(base_url=base_url, key_id=key_id) + auth_result = _poll_for_authentication( + base_url=base_url, key_id=key_id, poll_secret=poll_secret + ) if auth_result: api_key = auth_result["api_key"] user_id = auth_result["user_id"] - # Save token data (simplified for CLI - we just need the key) + # Save token data. base_url is stored so we can verify origin + # before reusing the key on a subsequent CLI invocation. save_token( { + "base_url": base_url.rstrip("/"), "key": api_key, "user_id": user_id or "cli-user", "user_email": "unknown", diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index 22de5a78614..be55f79c066 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -74,9 +74,10 @@ def cli(ctx: click.Context, base_url: str, api_key: Optional[str]) -> None: """LiteLLM Proxy CLI - Manage your LiteLLM proxy server""" ctx.ensure_object(dict) - # If no API key provided via flag or environment variable, try to load from saved token + # If no API key provided via flag or environment variable, try to load from saved token. + # Pass base_url so we only use the stored key when it was issued for this server. if api_key is None: - api_key = get_stored_api_key() + api_key = get_stored_api_key(expected_base_url=base_url) ctx.obj["base_url"] = base_url ctx.obj["api_key"] = api_key diff --git a/litellm/proxy/client/client.py b/litellm/proxy/client/client.py index 12b5cd79f79..929ad46a77c 100644 --- a/litellm/proxy/client/client.py +++ b/litellm/proxy/client/client.py @@ -28,12 +28,17 @@ class Client: api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token. timeout: Request timeout in seconds (default: 30) """ - self._base_url = base_url.rstrip("/") # Remove trailing slash if present - self._api_key = get_litellm_gateway_api_key() or api_key + self._base_url = base_url.rstrip("/") + # Only use the stored CLI key when it was issued for this server. + self._api_key = api_key or get_litellm_gateway_api_key( + expected_base_url=self._base_url + ) # Initialize resource clients - self.http = HTTPClient(base_url=base_url, api_key=api_key, timeout=timeout) + self.http = HTTPClient( + base_url=base_url, api_key=self._api_key, timeout=timeout + ) self.models = ModelsManagementClient( base_url=self._base_url, api_key=self._api_key ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 76c52f83ee4..438da037527 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -744,6 +744,11 @@ class ProxyBaseLLMRequestProcessing: "aingest", "aretrieve_container", "adelete_container", + "aupload_container_file", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", "acreate_skill", "alist_skills", "aget_skill", @@ -1001,6 +1006,11 @@ class ProxyBaseLLMRequestProcessing: "aingest", "aretrieve_container", "adelete_container", + "aupload_container_file", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", "acreate_skill", "alist_skills", "aget_skill", @@ -1594,6 +1604,12 @@ class ProxyBaseLLMRequestProcessing: # here would duplicate the guardrail API call # (e.g. double OpenAI Moderation charges). continue + if "async_post_call_streaming_iterator_hook" in type(cb).__dict__: + # Skip — the guardrail already scanned the assembled + # response via its own streaming iterator hook in the + # streaming pipeline. re running this function async_post_call_success_hook + # here would duplicate the scan and can spuriously block the guardrail that already passed / failed. + continue else: guardrail_result = await cb.async_post_call_success_hook( user_api_key_dict=captured_user_api_key_dict, diff --git a/litellm/proxy/common_utils/cache_coordinator.py b/litellm/proxy/common_utils/cache_coordinator.py index 24da9450ab8..abb0402d3b9 100644 --- a/litellm/proxy/common_utils/cache_coordinator.py +++ b/litellm/proxy/common_utils/cache_coordinator.py @@ -20,11 +20,27 @@ T = TypeVar("T") class AsyncCacheProtocol(Protocol): - """Protocol for cache backends used by EventDrivenCacheCoordinator.""" + """Protocol for cache backends used by EventDrivenCacheCoordinator. - async def async_get_cache(self, key: str, **kwargs: Any) -> Any: ... + Matches ``DualCache`` / ``UserApiKeyCache`` call shapes (explicit optional params + before ``**kwargs``), not only ``(key, **kwargs)``, so overloads validate. + """ - async def async_set_cache(self, key: str, value: Any, **kwargs: Any) -> Any: ... + async def async_get_cache( + self, + key: str, + parent_otel_span: Any = None, + local_only: bool = False, + **kwargs: Any, + ) -> Any: ... + + async def async_set_cache( + self, + key: str, + value: Any, + local_only: bool = False, + **kwargs: Any, + ) -> Any: ... class EventDrivenCacheCoordinator: @@ -36,6 +52,9 @@ class EventDrivenCacheCoordinator: - Other requests: wait for the signal, then read from cache. Create one instance per resource (e.g. one for global spend, one for feature flags). + + Args: + log_prefix: Prefix for debug log messages. """ def __init__(self, log_prefix: str = "[CACHE]"): diff --git a/litellm/proxy/common_utils/cache_pydantic_utils.py b/litellm/proxy/common_utils/cache_pydantic_utils.py new file mode 100644 index 00000000000..80a8d6281a1 --- /dev/null +++ b/litellm/proxy/common_utils/cache_pydantic_utils.py @@ -0,0 +1,93 @@ +""" +DualCache presents a single API for reads and writes, but the two backends behave +differently: the in-memory layer can store arbitrary Python objects (including live +``BaseModel`` instances), while Redis persists strings and therefore needs JSON-safe +payloads (``json.dumps`` on the Redis side). + +Call sites therefore see cache ``value`` / ``cached`` as effectively ``Any``: the same +key may deserialize to a model on one process (memory hit) or to a ``dict`` after a +Redis round-trip. ``CacheCodec`` centralizes encode/decode at that boundary: +``CacheCodec.serialize`` before ``set``, ``CacheCodec.deserialize`` after ``get`` +when you need a typed ``BaseModel``. + +``dataclasses`` are not supported: only ``dict`` and Pydantic ``BaseModel`` inputs +are encoded; pass a Pydantic model or convert with e.g. ``dataclasses.asdict`` first. +""" + +from __future__ import annotations + +from typing import Any, Optional, Type, TypeVar + +from pydantic import BaseModel, ValidationError + +from litellm._logging import verbose_proxy_logger + +T = TypeVar("T", bound=BaseModel) + + +class CacheCodec: + """ + Encode/decode Pydantic models for DualCache (memory vs Redis safe payloads). + + Dataclasses are not supported yet (only ``dict`` and ``BaseModel``). + + Use ``serialize`` with ``model_type`` when writing so the same schema is used + as on read (``deserialize``). Pass ``model_type`` whenever you know it + (validates ``dict`` payloads and normalizes ``BaseModel`` instances). + """ + + @staticmethod + def serialize(value: Any, model_type: Optional[Type[T]] = None) -> Any: + """ + Encode a value for DualCache / Redis (``json.dumps``-safe). + + If ``model_type`` is set, the payload is validated with that model, then + ``model_dump(mode="json", exclude_none=True)`` — symmetric with ``deserialize``. + + If the value is already an instance of ``model_type`` (or a subclass), + ``model_validate`` is skipped to avoid an unnecessary Pydantic copy — the + value is dumped directly. + + If ``model_type`` is omitted, any ``BaseModel`` is dumped as above; other + values (e.g. plain ``dict``) are returned unchanged. + """ + if model_type is not None: + if isinstance(value, model_type): + # Already the right type: dump directly, skip re-validation copy. + return value.model_dump(mode="json", exclude_none=True) + if isinstance(value, (dict, BaseModel)): + return model_type.model_validate(value).model_dump( + mode="json", exclude_none=True + ) + return value + if isinstance(value, BaseModel): + return value.model_dump(mode="json", exclude_none=True) + return value + + @staticmethod + def deserialize(cached: Any, model_type: Type[T]) -> Optional[T]: + """ + Decode a cache entry to ``model_type``. + + - ``None`` → ``None`` + - Already an instance of ``model_type`` (including subclasses) → returned as-is + - ``dict`` → ``model_type.model_validate(...)``; on ``ValidationError``, + logs a warning and returns ``None`` (treat as cache miss; avoids serving + malformed or schema-drifted entries) + - Any other type → ``None`` (caller should treat as cache miss or log) + """ + if cached is None: + return None + if isinstance(cached, model_type): + return cached + if isinstance(cached, dict): + try: + return model_type.model_validate(cached) + except ValidationError as e: + verbose_proxy_logger.warning( + "CacheCodec.deserialize: validation failed for %s (%s)", + model_type.__name__, + e, + ) + return None + return None diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 7ddd722a80e..198d9503cb0 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -14,6 +14,8 @@ from litellm.types.utils import ( blue_color_code = "\033[94m" reset_color_code = "\033[0m" +TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY = "_pillar_response_headers_trusted" + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -417,10 +419,19 @@ def get_logging_caching_headers(request_data: Dict) -> Optional[Dict]: if "semantic-similarity" in _metadata: headers["x-litellm-semantic-similarity"] = str(_metadata["semantic-similarity"]) + is_trusted_pillar_metadata = ( + _metadata.get(TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY) is True + ) pillar_headers = _metadata.get("pillar_response_headers") - if isinstance(pillar_headers, dict): - headers.update(pillar_headers) - elif "pillar_flagged" in _metadata: + if is_trusted_pillar_metadata and isinstance(pillar_headers, dict): + headers.update( + { + key: str(value) + for key, value in pillar_headers.items() + if isinstance(key, str) and key.lower().startswith("x-pillar-") + } + ) + elif is_trusted_pillar_metadata and "pillar_flagged" in _metadata: headers["x-pillar-flagged"] = str(_metadata["pillar_flagged"]).lower() return headers diff --git a/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py b/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py index c25d8533128..67a24567461 100644 --- a/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py +++ b/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py @@ -8,7 +8,7 @@ from datetime import datetime, timezone from typing import Any, Dict, List, Optional from litellm._logging import verbose_proxy_logger -from litellm.caching import DualCache +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.constants import ( EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME, LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE, @@ -31,7 +31,7 @@ class ExpiredUISessionKeyCleanupManager: def __init__( self, prisma_client: PrismaClient, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, pod_lock_manager=None, ): self.prisma_client = prisma_client diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index e486336cec0..0928ce914da 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -52,6 +52,37 @@ class ResetBudgetJob: ### RESET MULTI-WINDOW BUDGETS ### await self.reset_budget_windows() + @staticmethod + async def _invalidate_spend_counter(counter_key: str) -> None: + """Zero a spend counter so a DB-row reset takes effect immediately. + + Call AFTER the DB write commits. Clearing Redis before the DB + commit opens a window where get_current_spend reads 0 from Redis + while the DB still holds the pre-reset value, allowing bypass. + """ + try: + from litellm.proxy.proxy_server import spend_counter_cache + + spend_counter_cache.in_memory_cache.set_cache( + key=counter_key, value=0.0, ttl=60 + ) + if spend_counter_cache.redis_cache is not None: + try: + await spend_counter_cache.redis_cache.async_set_cache( + key=counter_key, value=0.0, ttl=60 + ) + except Exception as redis_err: + verbose_proxy_logger.warning( + "Failed to reset spend counter %s in Redis: %s. " + "Budget may be over-enforced until counter expires.", + counter_key, + redis_err, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to reset spend counter %s: %s", counter_key, e + ) + async def reset_budget_for_litellm_team_members( self, budgets_to_reset: List[LiteLLM_BudgetTableFull] ): @@ -64,46 +95,30 @@ class ResetBudgetJob: if budget.budget_id is not None ] - # Reset spend counters for affected team members. - # Reset Redis directly so a transient failure doesn't leave stale - # counters that get_current_spend would read as authoritative. try: - from litellm.proxy.proxy_server import spend_counter_cache - memberships = await self.prisma_client.db.litellm_teammembership.find_many( where={"budget_id": {"in": budget_ids}} ) - for m in memberships: - counter_key = f"spend:team_member:{m.user_id}:{m.team_id}" - # Always reset in-memory - spend_counter_cache.in_memory_cache.set_cache( - key=counter_key, value=0.0 - ) - # Explicitly reset Redis with warning on failure - if spend_counter_cache.redis_cache is not None: - try: - await spend_counter_cache.redis_cache.async_set_cache( - key=counter_key, value=0.0 - ) - except Exception as redis_err: - verbose_proxy_logger.warning( - "Failed to reset team member spend counter in Redis %s: %s. " - "Budget may be over-enforced until counter expires.", - counter_key, - redis_err, - ) except Exception as e: + memberships = [] verbose_proxy_logger.warning( - "Failed to reset team member spend counters: %s", e + "Failed to fetch team memberships for counter invalidation: %s", e ) - return await self.prisma_client.db.litellm_teammembership.update_many( + update_result = await self.prisma_client.db.litellm_teammembership.update_many( where={"budget_id": {"in": budget_ids}}, data={ "spend": 0, }, ) + for m in memberships: + await self._invalidate_spend_counter( + f"spend:team_member:{m.user_id}:{m.team_id}" + ) + + return update_result + async def reset_budget_for_keys_linked_to_budgets( self, budgets_to_reset: List[LiteLLM_BudgetTableFull] ): @@ -126,17 +141,36 @@ class ResetBudgetJob: if not budget_ids: return - return await self.prisma_client.db.litellm_verificationtoken.update_many( - where={ - "budget_id": {"in": budget_ids}, - "budget_duration": None, # only keys without their own reset schedule - "spend": {"gt": 0}, # only reset keys that have accumulated spend - }, - data={ - "spend": 0, - }, + where_clause: dict = { + "budget_id": {"in": budget_ids}, + "budget_duration": None, # only keys without their own reset schedule + "spend": {"gt": 0}, # only reset keys that have accumulated spend + } + + try: + keys = await self.prisma_client.db.litellm_verificationtoken.find_many( + where=where_clause + ) + except Exception as e: + keys = [] + verbose_proxy_logger.warning( + "Failed to fetch keys for counter invalidation: %s", e + ) + + update_result = ( + await self.prisma_client.db.litellm_verificationtoken.update_many( + where=where_clause, + data={ + "spend": 0, + }, + ) ) + for k in keys: + await self._invalidate_spend_counter(f"spend:key:{k.token}") + + return update_result + async def reset_budget_for_litellm_budget_table(self): """ Resets the budget for all LiteLLM End-Users (Customers), and Team Members if their budget has expired @@ -365,6 +399,10 @@ class ResetBudgetJob: data_list=updated_keys, table_name="key", ) + for k in updated_keys: + token = getattr(k, "token", None) + if token: + await self._invalidate_spend_counter(f"spend:key:{token}") end_time = time.time() if len(failed_keys) > 0: # If any keys failed to reset @@ -450,6 +488,12 @@ class ResetBudgetJob: data_list=updated_users, table_name="user", ) + for u in updated_users: + user_id = getattr(u, "user_id", None) + if user_id: + await self._invalidate_spend_counter( + f"spend:user:{user_id}" + ) end_time = time.time() if len(failed_users) > 0: # If any users failed to reset @@ -541,6 +585,12 @@ class ResetBudgetJob: data_list=updated_teams, table_name="team", ) + for t in updated_teams: + team_id = getattr(t, "team_id", None) + if team_id: + await self._invalidate_spend_counter( + f"spend:team:{team_id}" + ) end_time = time.time() if len(failed_teams) > 0: # If any teams failed to reset diff --git a/litellm/proxy/common_utils/static_asset_utils.py b/litellm/proxy/common_utils/static_asset_utils.py new file mode 100644 index 00000000000..c108af2b475 --- /dev/null +++ b/litellm/proxy/common_utils/static_asset_utils.py @@ -0,0 +1,52 @@ +"""Helpers for unauthenticated logo / favicon endpoints.""" + +import os +from typing import Optional, Tuple + +from litellm._logging import verbose_proxy_logger + +LOCAL_IMAGE_HEADER_BYTES = 512 + + +def detect_local_image_media_type(header: bytes) -> Optional[str]: + """Return a browser image media type for supported local image signatures.""" + if header[0:8] == b"\x89PNG\r\n\x1a\n": + return "image/png" + if header[0:4] == b"GIF8" and header[5:6] == b"a": + return "image/gif" + if header[0:3] == b"\xff\xd8\xff": + return "image/jpeg" + if header[0:4] == b"RIFF" and header[8:12] == b"WEBP": + return "image/webp" + if header[0:4] in (b"\x00\x00\x01\x00", b"\x00\x00\x02\x00"): + return "image/x-icon" + return None + + +def resolve_validated_local_image_path(candidate: str) -> Optional[Tuple[str, str]]: + """Resolve ``candidate`` only when it is an existing supported image file.""" + if not candidate: + return None + try: + resolved = os.path.realpath(os.path.expanduser(candidate)) + except (OSError, ValueError): + return None + if not os.path.isfile(resolved): + return None + + try: + with open(resolved, "rb") as f: + header = f.read(LOCAL_IMAGE_HEADER_BYTES) + except OSError as exc: + verbose_proxy_logger.debug("Could not read local asset %r: %s", candidate, exc) + return None + + media_type = detect_local_image_media_type(header) + if media_type is None: + verbose_proxy_logger.warning( + "Local asset %r is not a supported image file; falling back to default.", + candidate, + ) + return None + + return resolved, media_type diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py new file mode 100644 index 00000000000..914be364579 --- /dev/null +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from typing import Any, Optional, Type, TypeVar, Union, cast, overload + +from pydantic import BaseModel + +from litellm._logging import verbose_proxy_logger +from litellm.caching.dual_cache import DualCache +from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec + +T = TypeVar("T", bound=BaseModel) + + +class UserApiKeyCache(DualCache): + """ + DualCache wrapper for UserAPIKeyAuth-like payloads. + + Stores a Redis-safe JSON payload in BOTH in-memory and Redis to avoid + "memory returns BaseModel, Redis returns dict" format drift. + + When ``model_type`` is provided: + - writes are serialized via ``CacheCodec.serialize(..., model_type=...)`` + - reads are deserialized via ``CacheCodec.deserialize(..., model_type)`` + and return ``Optional[T]``: the model on success, ``None`` on cache miss + **or** if the cached payload fails validation (schema drift). On + validation failure after a cache hit, an error line is emitted via + ``verbose_proxy_logger``. + + When ``model_type`` is omitted, the interface behaves like ``DualCache``: + raw cached payload is returned (dict/str/etc.). + + ``async_set_cache_pipeline`` applies the same untyped Codec pass as omitting + ``model_type`` on ``async_set_cache`` (so ``BaseModel`` rows are dumped before Redis). + + ``get_cache`` / ``async_get_cache`` overloads and implementations must be contiguous + (no other methods in between) so mypy resolves ``@overload`` + implementation correctly. + """ + + @overload + def get_cache( + self, + key: Any, + parent_otel_span: Any = None, + local_only: bool = False, + *, + model_type: Type[T], + **kwargs: Any, + ) -> Optional[T]: ... + + @overload + def get_cache( + self, + key: Any, + parent_otel_span: Any = None, + local_only: bool = False, + **kwargs: Any, + ) -> Any: ... + + def get_cache( # type: ignore[override] + self, + key, + parent_otel_span=None, + local_only: bool = False, + model_type: Optional[Type[BaseModel]] = None, + **kwargs, + ) -> Union[Any, Optional[BaseModel]]: + if model_type is None and "model_type" in kwargs: + model_type = cast(Optional[Type[BaseModel]], kwargs.pop("model_type", None)) + cached = super().get_cache( + key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs + ) + if model_type is None: + return cached + if cached is None: + return None + decoded = CacheCodec.deserialize(cached, model_type=model_type) + if decoded is None: + verbose_proxy_logger.error( + "UserApiKeyCache.get_cache failed to deserialize cached value for " + "key=%r model_type=%s", + key, + getattr(model_type, "__name__", str(model_type)), + ) + return None + return decoded + + @overload + async def async_get_cache( + self, + key: Any, + parent_otel_span: Any = None, + local_only: bool = False, + *, + model_type: Type[T], + **kwargs: Any, + ) -> Optional[T]: ... + + @overload + async def async_get_cache( + self, + key: Any, + parent_otel_span: Any = None, + local_only: bool = False, + **kwargs: Any, + ) -> Any: ... + + async def async_get_cache( # type: ignore[override] + self, + key, + parent_otel_span=None, + local_only: bool = False, + model_type: Optional[Type[BaseModel]] = None, + **kwargs, + ) -> Union[Any, Optional[BaseModel]]: + if model_type is None and "model_type" in kwargs: + model_type = cast(Optional[Type[BaseModel]], kwargs.pop("model_type", None)) + cached = await super().async_get_cache( + key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs + ) + if model_type is None: + return cached + if cached is None: + return None + decoded = CacheCodec.deserialize(cached, model_type=model_type) + if decoded is None: + verbose_proxy_logger.error( + "UserApiKeyCache.async_get_cache failed to deserialize cached value for " + "key=%r model_type=%s", + key, + getattr(model_type, "__name__", str(model_type)), + ) + return None + return decoded + + def set_cache(self, key, value, local_only: bool = False, **kwargs): # type: ignore[override] + model_type = cast(Optional[Type[BaseModel]], kwargs.pop("model_type", None)) + payload = CacheCodec.serialize(value, model_type=model_type) + return super().set_cache( + key=key, value=payload, local_only=local_only, **kwargs + ) + + async def async_set_cache(self, key, value, local_only: bool = False, **kwargs): # type: ignore[override] + model_type = cast(Optional[Type[BaseModel]], kwargs.pop("model_type", None)) + payload = CacheCodec.serialize(value, model_type=model_type) + return await super().async_set_cache( + key=key, value=payload, local_only=local_only, **kwargs + ) + + async def async_set_cache_pipeline( # type: ignore[override] + self, cache_list: list, local_only: bool = False, **kwargs + ) -> None: + """ + Batch writes with the same Codec boundary as ``async_set_cache`` without + ``model_type``: ``BaseModel`` values become JSON-safe dicts; dicts/scalars unchanged. + """ + normalized = [ + (key, CacheCodec.serialize(value, model_type=None)) + for key, value in cache_list + ] + return await super().async_set_cache_pipeline( + cache_list=normalized, local_only=local_only, **kwargs + ) diff --git a/litellm/proxy/container_endpoints/handler_factory.py b/litellm/proxy/container_endpoints/handler_factory.py index fae7f939aed..794051e90f8 100644 --- a/litellm/proxy/container_endpoints/handler_factory.py +++ b/litellm/proxy/container_endpoints/handler_factory.py @@ -19,7 +19,6 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( get_custom_llm_provider_from_request_headers, get_custom_llm_provider_from_request_query, ) -from litellm.responses.utils import ResponsesAPIRequestUtils def _load_endpoints_config() -> Dict: @@ -64,10 +63,12 @@ def _create_handler_for_path_params( request: Request, container_id: str, file_id: str, + fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): return await _process_binary_request( request=request, + fastapi_response=fastapi_response, container_id=container_id, file_id=file_id, user_api_key_dict=user_api_key_dict, @@ -152,63 +153,61 @@ def _create_handler_for_path_params( async def _process_binary_request( request: Request, + fastapi_response: Response, container_id: str, file_id: str, user_api_key_dict: UserAPIKeyAuth, ): """ - Process binary content requests using the proper transformation pattern. + Process binary content requests through the standard proxy/router pipeline. - This uses the provider config transformations and llm_http_handler - to maintain consistency with the established pattern. + The router owns managed container ID decoding and deployment selection. This + handler only adapts the byte response to FastAPI. """ - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler - from litellm.types.router import GenericLiteLLMParams + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) - # Extract custom_llm_provider custom_llm_provider = ( get_custom_llm_provider_from_request_headers(request=request) or get_custom_llm_provider_from_request_query(request=request) or "openai" ) - - # Build litellm_params - credentials are resolved by provider config from env - litellm_params = GenericLiteLLMParams() - - # Decode container ID and extract provider info - decoded = ResponsesAPIRequestUtils._decode_container_id(container_id) - original_container_id = decoded.get("response_id", container_id) - - # If container ID has encoded provider info and user didn't explicitly set provider, use it - decoded_provider = decoded.get("custom_llm_provider") - if decoded_provider and custom_llm_provider == "openai": - custom_llm_provider = decoded_provider - - # Get the provider config - container_provider_config = _get_container_provider_config(custom_llm_provider) - - # Create logging object - logging_obj = Logging( - model="container-file-content", - messages=[], - stream=False, - call_type="container_file_content", - start_time=None, - litellm_call_id="", - function_id="", - ) - - # Use the HTTP handler to make the request - handler = BaseLLMHTTPHandler() + data: Dict[str, Any] = { + "container_id": container_id, + "file_id": file_id, + "custom_llm_provider": custom_llm_provider, + } + processor = ProxyBaseLLMRequestProcessing(data=data) try: - content = await handler.async_container_file_content_handler( - container_id=original_container_id, # Use decoded original ID - file_id=file_id, - container_provider_config=container_provider_config, - litellm_params=litellm_params, - logging_obj=logging_obj, + content = await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="aretrieve_container_file_content", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, ) # Determine content type based on common file extensions in the file_id @@ -229,13 +228,25 @@ async def _process_binary_request( elif ".pdf" in file_id_lower: content_type = "application/pdf" + if not isinstance(content, bytes): + raise TypeError( + "aretrieve_container_file_content expected bytes, got " + f"{type(content).__name__}" + ) + return Response( content=content, + headers=dict(fastapi_response.headers), media_type=content_type, ) except Exception as e: - raise e + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) async def _process_multipart_upload_request( @@ -284,16 +295,7 @@ async def _process_multipart_upload_request( or "openai" ) - # Decode container ID and extract provider info - decoded = ResponsesAPIRequestUtils._decode_container_id(container_id) - original_container_id = decoded.get("response_id", container_id) - - # If container ID has encoded provider info and user didn't explicitly set provider, use it - decoded_provider = decoded.get("custom_llm_provider") - if decoded_provider and custom_llm_provider == "openai": - custom_llm_provider = decoded_provider - - data["container_id"] = original_container_id # Use decoded original ID + data["container_id"] = container_id data["custom_llm_provider"] = custom_llm_provider processor = ProxyBaseLLMRequestProcessing(data=data) @@ -359,21 +361,6 @@ async def _process_request( or "openai" ) - # Decode container_id if present in path_params - if "container_id" in path_params: - decoded = ResponsesAPIRequestUtils._decode_container_id( - path_params["container_id"] - ) - original_container_id = decoded.get("response_id", path_params["container_id"]) - - # If container ID has encoded provider info and user didn't explicitly set provider, use it - decoded_provider = decoded.get("custom_llm_provider") - if decoded_provider and custom_llm_provider == "openai": - custom_llm_provider = decoded_provider - - # Update path_params with decoded original ID - data["container_id"] = original_container_id - data["custom_llm_provider"] = custom_llm_provider processor = ProxyBaseLLMRequestProcessing(data=data) diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index bf60a087c65..a979471dc8e 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -129,7 +129,9 @@ class SpendCounterReseed: """ lock = await SpendCounterReseed._get_lock(counter_key) async with lock: - # Re-check after acquiring the lock - another waiter may have warmed it. + # Re-check after acquiring the lock. Skip in-memory on a clean + # Redis miss - in-memory is per-pod-stale. + redis_clean_miss = False if spend_counter_cache.redis_cache is not None: try: val = await spend_counter_cache.redis_cache.async_get_cache( @@ -137,11 +139,13 @@ class SpendCounterReseed: ) if val is not None: return float(val) + redis_clean_miss = True except Exception: pass - val = spend_counter_cache.in_memory_cache.get_cache(key=counter_key) - if val is not None: - return float(val) + if not redis_clean_miss: + val = spend_counter_cache.in_memory_cache.get_cache(key=counter_key) + if val is not None: + return float(val) db_spend = await SpendCounterReseed.from_db(prisma_client, counter_key) if db_spend is None: @@ -149,7 +153,7 @@ class SpendCounterReseed: # Warm even when 0 so subsequent reads hit cache, not DB. try: await spend_counter_cache.async_increment_cache( - key=counter_key, value=db_spend + key=counter_key, value=db_spend, refresh_ttl=True ) except Exception: verbose_proxy_logger.exception( diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 6ada8f58783..967ac9f0ac4 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -1,10 +1,6 @@ -from datetime import datetime +from fastapi import APIRouter, Depends, Request, Response +from fastapi.responses import ORJSONResponse -from fastapi import APIRouter, Depends, HTTPException, Request, Response -from fastapi.responses import ORJSONResponse, StreamingResponse - -import litellm -from litellm._uuid import uuid from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -30,12 +26,17 @@ async def google_generate_content( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): - from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.proxy_server import ( general_settings, llm_router, proxy_config, proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, version, ) @@ -43,48 +44,33 @@ async def google_generate_content( if "model" not in data: data["model"] = model_name - # Extract generationConfig and pass it as config parameter - generation_config = data.pop("generationConfig", None) - if generation_config: - data["config"] = generation_config - - # Add user authentication metadata for cost tracking - data = await add_litellm_data_to_request( - data=data, - request=request, - user_api_key_dict=user_api_key_dict, - proxy_config=proxy_config, - general_settings=general_settings, - version=version, - ) - - # Create logging object with full request metadata so callbacks (e.g. S3) get user/trace_id - data["litellm_call_id"] = request.headers.get( - "x-litellm-call-id", str(uuid.uuid4()) - ) - logging_obj, data = litellm.utils.function_setup( - original_function="agenerate_content", - rules_obj=litellm.utils.Rules(), - start_time=datetime.now(), - **data, - ) - data["litellm_logging_obj"] = logging_obj - - # call router - if llm_router is None: - raise HTTPException(status_code=500, detail="Router not initialized") - response = await llm_router.agenerate_content(**data) - success_headers = await ProxyBaseLLMRequestProcessing.build_litellm_proxy_success_headers_from_llm_response( - response=response, - request_data=data, - request=request, - user_api_key_dict=user_api_key_dict, - logging_obj=logging_obj, - version=version, - proxy_logging_obj=proxy_logging_obj, - ) - fastapi_response.headers.update(success_headers) - return response + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="agenerate_content", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=model_name, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) @router.post( @@ -101,73 +87,52 @@ async def google_stream_generate_content( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): - from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.proxy_server import ( general_settings, llm_router, proxy_config, proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, version, ) data = await _read_request_body(request=request) - if "model" not in data: data["model"] = model_name + data["stream"] = True - data["stream"] = True # enforce streaming for this endpoint - - # Extract generationConfig and pass it as config parameter - generation_config = data.pop("generationConfig", None) - if generation_config: - data["config"] = generation_config - - # Add user authentication metadata for cost tracking - data = await add_litellm_data_to_request( - data=data, - request=request, - user_api_key_dict=user_api_key_dict, - proxy_config=proxy_config, - general_settings=general_settings, - version=version, - ) - - # Create logging object with full request metadata so streaming END callbacks (e.g. S3) get user/trace_id - data["litellm_call_id"] = request.headers.get( - "x-litellm-call-id", str(uuid.uuid4()) - ) - logging_obj, data = litellm.utils.function_setup( - original_function="agenerate_content_stream", - rules_obj=litellm.utils.Rules(), - start_time=datetime.now(), - **data, - ) - data["litellm_logging_obj"] = logging_obj - - # call router - if llm_router is None: - raise HTTPException(status_code=500, detail="Router not initialized") - response = await llm_router.agenerate_content_stream(**data) - - success_headers = await ProxyBaseLLMRequestProcessing.build_litellm_proxy_success_headers_from_llm_response( - response=response, - request_data=data, - request=request, - user_api_key_dict=user_api_key_dict, - logging_obj=logging_obj, - version=version, - proxy_logging_obj=proxy_logging_obj, - ) - - # Check if response is an async iterator (streaming response) - if response is not None and hasattr(response, "__aiter__"): - return StreamingResponse( - content=response, - media_type="text/event-stream", - headers=success_headers, + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="agenerate_content_stream", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=model_name, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, ) - fastapi_response.headers.update(success_headers) - return response @router.post( diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index e5df26b7c2c..bb1db3d62d2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -71,6 +71,7 @@ from litellm.types.utils import ( ) GUARDRAIL_NAME = "bedrock" +_BEDROCK_DYNAMIC_BODY_DENYLIST = frozenset({"content", "source"}) class GuardrailMessageFilterResult(NamedTuple): @@ -413,11 +414,18 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) api_key: Optional[str] = None if request_data: - bedrock_request_data.update( + dynamic_request_body_params = ( self.get_guardrail_dynamic_request_body_params( request_data=request_data ) ) + bedrock_request_data.update( + { + key: value + for key, value in dynamic_request_body_params.items() + if key not in _BEDROCK_DYNAMIC_BODY_DENYLIST + } + ) if request_data.get("api_key") is not None: api_key = request_data["api_key"] diff --git a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py index 1b3f11e56f9..c9c73053a0e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py @@ -29,6 +29,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( + TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY, add_guardrail_to_applied_guardrails_header, get_metadata_variable_name_from_kwargs, ) @@ -144,6 +145,7 @@ def build_pillar_response_headers(metadata_store: Dict[str, Any]) -> Dict[str, s if headers: metadata_store["pillar_response_headers"] = headers + metadata_store[TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY] = True return headers diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index 6dd0288cb09..37be832d350 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -225,10 +225,10 @@ class ToolPermissionGuardrail(CustomGuardrail): def _parse_tool_call_arguments( self, tool_call: ChatCompletionMessageToolCall - ) -> Dict[str, Any]: + ) -> tuple[Optional[Dict[str, Any]], Optional[str]]: arguments = getattr(tool_call.function, "arguments", None) if not arguments: - return {} + return None, "missing arguments" parsed_arguments: Any = {} try: @@ -236,22 +236,24 @@ class ToolPermissionGuardrail(CustomGuardrail): parsed_arguments = json.loads(arguments) elif isinstance(arguments, dict): parsed_arguments = arguments - except json.JSONDecodeError as exc: + else: + return None, "arguments must be a JSON object" + except (json.JSONDecodeError, TypeError) as exc: verbose_proxy_logger.warning( "Tool Permission Guardrail: Failed to decode arguments for tool %s: %s", tool_call.function.name, exc, ) - return {} + return None, "arguments could not be parsed" if isinstance(parsed_arguments, dict): - return parsed_arguments + return parsed_arguments, None verbose_proxy_logger.debug( - "Tool Permission Guardrail: Ignoring non-dict arguments for tool %s", + "Tool Permission Guardrail: Rejecting non-dict arguments for tool %s", tool_call.function.name, ) - return {} + return None, "arguments must be a JSON object" def _collect_argument_paths( self, @@ -331,10 +333,21 @@ class ToolPermissionGuardrail(CustomGuardrail): continue if rule.allowed_param_patterns and should_check_params: - arguments = self._parse_tool_call_arguments(tool_call) + arguments, parse_error = self._parse_tool_call_arguments(tool_call) + if parse_error: + default_message = f"Tool '{tool_identifier}' {parse_error} required by rule '{rule.id}'" + message = self.render_violation_message( + default=default_message, + context={"tool_name": tool_identifier, "rule_id": rule.id}, + ) + return False, rule.id, message if not arguments: - last_pattern_failure_msg = f"Tool '{tool_identifier}' is missing arguments required by rule '{rule.id}'" - continue + default_message = f"Tool '{tool_identifier}' is missing arguments required by rule '{rule.id}'" + message = self.render_violation_message( + default=default_message, + context={"tool_name": tool_identifier, "rule_id": rule.id}, + ) + return False, rule.id, message patterns_match, failure_message = self._patterns_match_for_rule( arguments=arguments, @@ -365,6 +378,33 @@ class ToolPermissionGuardrail(CustomGuardrail): ) return is_allowed, None, message + @staticmethod + def _get_mapping_value(item: Any, key: str) -> Any: + if isinstance(item, dict): + return item.get(key) + return getattr(item, key, None) + + @staticmethod + def _legacy_function_call_id(choice_index: int) -> str: + return f"legacy_function_call_{choice_index}" + + def _legacy_function_call_to_tool_call( + self, function_call: Any, choice_index: int + ) -> Optional[ChatCompletionMessageToolCall]: + if function_call is None: + return None + + function_name = self._get_mapping_value(function_call, "name") + arguments = self._get_mapping_value(function_call, "arguments") or "" + if not function_name: + return None + + return ChatCompletionMessageToolCall( + id=self._legacy_function_call_id(choice_index), + type="function", + function={"name": function_name, "arguments": arguments}, + ) + def _extract_tool_calls_from_response( self, response: ModelResponse ) -> List[ChatCompletionMessageToolCall]: @@ -379,13 +419,72 @@ class ToolPermissionGuardrail(CustomGuardrail): """ tool_calls = [] - for choice in response.choices: + for choice_index, choice in enumerate(response.choices): if isinstance(choice, Choices): for tool in choice.message.tool_calls or []: tool_calls.append(tool) + legacy_tool_call = self._legacy_function_call_to_tool_call( + getattr(choice.message, "function_call", None), choice_index + ) + if legacy_tool_call is not None: + tool_calls.append(legacy_tool_call) return tool_calls + def _get_request_tool_name(self, tool: Any) -> tuple[Optional[str], Optional[str]]: + tool_type = self._get_mapping_value(tool, "type") + if tool_type != "function": + return None, tool_type + + function = self._get_mapping_value(tool, "function") + tool_name = self._get_mapping_value(function, "name") + return tool_name, tool_type + + def _get_legacy_function_name(self, function: Any) -> Optional[str]: + return self._get_mapping_value(function, "name") + + def _get_named_tool_choice(self, data: dict) -> Optional[str]: + tool_choice = data.get("tool_choice") + if not tool_choice or tool_choice in ("auto", "none", "required"): + return None + if isinstance(tool_choice, str): + return tool_choice + if self._get_mapping_value(tool_choice, "type") != "function": + return None + return self._get_mapping_value( + self._get_mapping_value(tool_choice, "function"), "name" + ) + + def _get_named_function_call(self, data: dict) -> Optional[str]: + function_call = data.get("function_call") + if not function_call or function_call in ("auto", "none"): + return None + if isinstance(function_call, str): + return function_call + return self._get_mapping_value(function_call, "name") + + def _collect_request_tools(self, data: dict) -> List[tuple[str, Optional[str]]]: + request_tools: List[tuple[str, Optional[str]]] = [] + + for tool in data.get("tools") or []: + tool_name, tool_type = self._get_request_tool_name(tool) + if tool_name is not None: + request_tools.append((tool_name, tool_type)) + + for function in data.get("functions") or []: + function_name = self._get_legacy_function_name(function) + if function_name is not None: + request_tools.append((function_name, "function")) + + for forced_tool_name in ( + self._get_named_tool_choice(data), + self._get_named_function_call(data), + ): + if forced_tool_name is not None: + request_tools.append((forced_tool_name, "function")) + + return request_tools + def _modify_request_with_permission_errors( self, data: dict, @@ -410,19 +509,32 @@ class ToolPermissionGuardrail(CustomGuardrail): for tool_use in denied_tool_names: error_tool_names.add(tool_use) - # Modify the tools tools: Optional[List[ChatCompletionToolParam]] = data.get("tools") - if tools is None: - return data - - new_tools = [] - for tool in tools: - if tool["type"] != "function": - continue - tool_name: str = tool["function"]["name"] - if tool_name not in error_tool_names: + if tools is not None: + new_tools = [] + for tool in tools: + tool_name, tool_type = self._get_request_tool_name(tool) + if tool_type == "function" and tool_name in error_tool_names: + continue new_tools.append(tool) - data["tools"] = new_tools + data["tools"] = new_tools + + functions = data.get("functions") + if functions is not None: + data["functions"] = [ + function + for function in functions + if self._get_legacy_function_name(function) not in error_tool_names + ] + + named_tool_choice = self._get_named_tool_choice(data) + if named_tool_choice in error_tool_names: + data["tool_choice"] = "none" + + named_function_call = self._get_named_function_call(data) + if named_function_call in error_tool_names: + data["function_call"] = "none" + return data def _create_permission_error_result( @@ -472,7 +584,7 @@ class ToolPermissionGuardrail(CustomGuardrail): error_results[tool_use.id] = error_result # Modify the response content - for choice in response.choices: + for choice_index, choice in enumerate(response.choices): if isinstance(choice, Choices): filtered_tool_calls = [] error_messages = [] @@ -490,6 +602,15 @@ class ToolPermissionGuardrail(CustomGuardrail): filtered_tool_calls if filtered_tool_calls else None ) + legacy_tool_call = self._legacy_function_call_to_tool_call( + getattr(choice.message, "function_call", None), choice_index + ) + if legacy_tool_call is not None: + legacy_error_result = error_results.get(legacy_tool_call.id) + if legacy_error_result is not None: + choice.message.function_call = None + error_messages.append(legacy_error_result.content) + # Add error messages to content if error_messages: existing_content = choice.message.content @@ -519,21 +640,16 @@ class ToolPermissionGuardrail(CustomGuardrail): if self.should_run_guardrail(data=data, event_type=event_type) is not True: return data - new_tools: Optional[List[ChatCompletionToolParam]] = data.get("tools") - if new_tools is None: + new_tools = self._collect_request_tools(data) + if not new_tools: verbose_proxy_logger.warning( - "Tool Permission Guardrail: not running guardrail. No tools in data" + "Tool Permission Guardrail: not running guardrail. No tools or functions in data" ) return data # Check permissions for each tool denied_tool_names = [] - for tool in new_tools: - if tool["type"] != "function": - continue - tool_name: str = tool["function"]["name"] - tool_type: Optional[str] = tool.get("type") - + for tool_name, tool_type in new_tools: is_allowed, _, message = self._check_tool_permission(tool_name, tool_type) if not is_allowed and message is not None: diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 7d67750c78f..7c340ff5df6 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -29,6 +29,10 @@ ILLEGAL_DISPLAY_PARAMS = [ "exception", # internal; not JSON-serializable, never for display "litellm_metadata", # internal tracking metadata with auth objects; not for display ] +# Provider routing fields. Allowed for proxy admins so they can see which +# region/version a deployment is checking; gated at the endpoint layer for +# non-admin callers (see _strip_admin_only_fields_from_health_result). +ADMIN_ONLY_HEALTH_DISPLAY_PARAMS = ("api_base", "api_version") MINIMAL_DISPLAY_PARAMS = ["model", "mode_error"] diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index b4b5de1746e..35c9edb937d 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -20,6 +20,7 @@ from litellm.proxy._types import ( CallInfo, EnterpriseLicenseData, Litellm_EntityType, + LitellmUserRoles, ProxyErrorTypes, ProxyException, UserAPIKeyAuth, @@ -28,6 +29,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.health_check import ( + ADMIN_ONLY_HEALTH_DISPLAY_PARAMS, _clean_endpoint_data, _update_litellm_params_for_health_check, perform_health_check, @@ -723,6 +725,129 @@ async def _save_background_health_checks_to_db( # Continue execution - don't let database save failure break health checks +_PROXY_ADMIN_ROLES = frozenset( + { + LitellmUserRoles.PROXY_ADMIN.value, + # View-only admins are operators (oncall, support); they need the + # routing fields (api_base, api_version) to diagnose health and tell + # which provider region a check is hitting. They cannot mutate config + # so granting them the read-only view is safe. + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + } +) + + +def _is_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> bool: + """ + Return True if the caller has a proxy-admin role (full or view-only). + + user_role on UserAPIKeyAuth can be either a LitellmUserRoles enum or its + string value depending on how the auth path constructed the object, so we + compare against the raw value rather than the enum identity. + """ + role = user_api_key_dict.user_role + if role is None: + return False + role_value = role.value if hasattr(role, "value") else role + return role_value in _PROXY_ADMIN_ROLES + + +def _strip_admin_only_fields_from_health_result(result: dict) -> dict: + """ + Return a copy of the /health response with provider routing fields + (``api_base``, ``api_version``) removed from each healthy/unhealthy + endpoint entry. Used to hide those fields from non-admin callers while + still showing them which deployments they own and whether each one is + healthy. Proxy admins receive the unmodified result. + """ + out = dict(result) + drop = set(ADMIN_ONLY_HEALTH_DISPLAY_PARAMS) + for key in ("healthy_endpoints", "unhealthy_endpoints"): + eps = out.get(key) + if isinstance(eps, list): + out[key] = [ + ( + {k: v for k, v in ep.items() if k not in drop} + if isinstance(ep, dict) + else ep + ) + for ep in eps + ] + return out + + +def _resolve_targeted_model_ids( + model_list: list, model: Optional[str], model_id: Optional[str] +) -> Optional[set]: + """ + Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of + deployment IDs the response should be scoped to. + + Mirrors the live-path semantics in ``perform_health_check()``: ``model`` + matches either the deployment's ``model_name`` alias or its + ``litellm_params.model`` provider string. ``model_id`` matches + ``model_info.id``. + + Both query params are validated against the supplied ``model_list``. + Callers pass an already-scoped list (filtered to the caller's allowed + models for non-admins, full list for admins), so a ``model_id`` that + isn't present resolves to an empty set rather than a single-element + set — preventing a non-admin from reading another deployment's cached + health entry by guessing its ID. + + Returns ``None`` when no targeting is requested — callers should treat + that as "no filter." + """ + if not model and not model_id: + return None + target_ids: set = set() + for m in model_list: + deployment_id = (m.get("model_info") or {}).get("id") + if not deployment_id: + continue + if model_id and deployment_id == model_id: + target_ids.add(deployment_id) + continue + if model: + litellm_model = (m.get("litellm_params") or {}).get("model") + if m.get("model_name") == model or litellm_model == model: + target_ids.add(deployment_id) + return target_ids + + +def _filter_health_check_results_by_model_ids( + results: dict, allowed_model_ids: set +) -> dict: + """ + Restrict a cached background health-check result dict to endpoints whose + model_id is in ``allowed_model_ids``. + + Endpoints without a model_id (e.g. CLI-model entries that predate the + model_id wiring) are dropped conservatively — we cannot prove they belong + to the caller, so they are excluded rather than leaked. + + Each retained endpoint is shallow-copied before being returned, so any + downstream transform (e.g. _strip_admin_only_fields_from_health_result) + cannot accidentally mutate the shared ``health_check_results`` cache. + """ + healthy = [ + dict(ep) + for ep in (results.get("healthy_endpoints") or []) + if ep.get("model_id") in allowed_model_ids + ] + unhealthy = [ + dict(ep) + for ep in (results.get("unhealthy_endpoints") or []) + if ep.get("model_id") in allowed_model_ids + ] + return { + "healthy_endpoints": healthy, + "unhealthy_endpoints": unhealthy, + "healthy_count": len(healthy), + "unhealthy_count": len(unhealthy), + } + + async def _perform_health_check_and_save( model_list, target_model, @@ -771,6 +896,7 @@ async def _perform_health_check_and_save( @router.get("/health", tags=["health"], dependencies=[Depends(user_api_key_auth)]) async def health_endpoint( + response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), model: Optional[str] = fastapi.Query( None, description="Specify the model name (optional)" @@ -838,11 +964,33 @@ async def health_endpoint( detail={"error": f"Model with ID {model_id} not found"}, ) + is_admin = _is_proxy_admin(user_api_key_dict) + model_specific_request = bool(model or model_id) + + def _post_process(result: dict) -> dict: + # api_base / api_version reveal which provider/region/internal host the + # deployment talks to; only proxy admins receive them. Non-admin keys + # still see model/model_id and the healthy/unhealthy status. We also + # set a header so non-admin clients that previously parsed those + # fields can detect the change programmatically. + # When a caller asked about a specific model/model_id and zero + # endpoints came back healthy, surface that as a 503 so monitoring + # systems can rely on the HTTP status instead of having to parse the + # body. The body shape is unchanged. + if model_specific_request and result.get("healthy_count", 0) == 0: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + if is_admin: + return result + response.headers["Litellm-Health-Field-Notice"] = ( + "api_base and api_version are admin-only on this endpoint" + ) + return _strip_admin_only_fields_from_health_result(result) + try: if llm_model_list is None: # if no router set, check if user set a model using litellm --model ollama/llama2 if user_model is not None: - return await _perform_health_check_and_save( + cli_result = await _perform_health_check_and_save( model_list=[], target_model=None, cli_model=user_model, @@ -853,20 +1001,81 @@ async def health_endpoint( model_id=None, # CLI model doesn't have model_id max_concurrency=health_check_concurrency, ) + return _post_process(cli_result) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": "Model list not initialized"}, ) _llm_model_list = copy.deepcopy(llm_model_list) ### FILTER MODELS FOR ONLY THOSE USER HAS ACCESS TO ### + # Live path: scope by model_name (every deployment has one). + # Cache path: scope by model_id (the cache is keyed on model_id). + # Consequence: a deployment whose model_name the caller can access + # but which lacks model_info.id will appear in the live /health + # response but NOT in the background-cache /health response. This is + # surfaced via the "warnings" field below so operators can fix the + # missing model_info.id rather than guess at the discrepancy. if len(user_api_key_dict.models) > 0: - pass - else: - pass # + allowed_models = set(user_api_key_dict.models) + _llm_model_list = [ + m for m in _llm_model_list if m.get("model_name") in allowed_models + ] if use_background_health_checks: - return health_check_results + # The cached background result covers every model. When the + # caller targets a specific model/model_id we have to narrow the + # cache to that deployment before _post_process evaluates + # healthy_count, otherwise an unhealthy "foo" combined with any + # other healthy model would still report healthy_count > 0 and + # the targeted-503 path would never fire. + targeted_ids = _resolve_targeted_model_ids(_llm_model_list, model, model_id) + if len(user_api_key_dict.models) > 0: + allowed_model_ids = { + (m.get("model_info") or {}).get("id") + for m in _llm_model_list + if (m.get("model_info") or {}).get("id") + } + # _llm_model_list is already scoped to the caller's allowed + # model_names above, so targeted_ids is implicitly the + # intersection of "targeted" and "allowed." + filter_ids = ( + targeted_ids if targeted_ids is not None else allowed_model_ids + ) + filtered = _filter_health_check_results_by_model_ids( + health_check_results, filter_ids + ) + if targeted_ids is None and not allowed_model_ids: + # Caller has accessible model_names but none of the + # matching deployments expose a model_info.id, so the + # cache filter (which keys on model_id) drops every + # entry. Surface this both as a warning log and a + # structured "warnings" field on the response so the + # caller can distinguish "no deployments found" from + # "deployments excluded due to missing model_info.id". + verbose_proxy_logger.warning( + "health_endpoint: scoped key %s has accessible models %s " + "but none of the matching deployments carry a model_info.id; " + "background health-check cache will return an empty result.", + user_api_key_dict.user_id, + list(user_api_key_dict.models), + ) + filtered["warnings"] = [ + "Some accessible deployments are missing model_info.id " + "and were excluded from this response. Ask a proxy admin " + "to populate model_info.id for these models." + ] + return _post_process(filtered) + if targeted_ids is not None: + # Admin caller targeting a specific model: filter the cache + # so the response (and the targeted-503 check) reflects only + # that deployment, not the global aggregate. + return _post_process( + _filter_health_check_results_by_model_ids( + health_check_results, targeted_ids + ) + ) + return _post_process(health_check_results) else: - return await _perform_health_check_and_save( + router_result = await _perform_health_check_and_save( model_list=_llm_model_list, target_model=target_model, cli_model=None, @@ -877,6 +1086,7 @@ async def health_endpoint( model_id=model_id, max_concurrency=health_check_concurrency, ) + return _post_process(router_result) except Exception as e: verbose_proxy_logger.error( "litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - {}".format( @@ -1242,7 +1452,7 @@ def callback_name(callback): tags=["health"], dependencies=[Depends(user_api_key_auth)], ) -async def health_readiness(): +async def health_readiness(response: Response): """ Unprotected endpoint for checking if worker can receive requests """ @@ -1275,8 +1485,8 @@ async def health_readiness(): try: index_info = await litellm.cache.cache._index_info() except Exception as e: - index_info = "index does not exist - error: " + str(e) - cache_type = {"type": cache_type, "index_info": index_info} + index_info = "index does not exist - error: " + str(e) # type: ignore[assignment] + cache_type = {"type": cache_type, "index_info": index_info} # type: ignore[assignment] # check log level log_level_name = logging.getLevelName(verbose_logger.getEffectiveLevel()) @@ -1285,6 +1495,12 @@ async def health_readiness(): # check DB if prisma_client is not None: # if db passed in, check if it's connected db_health_status = await _db_health_readiness_check() + # A configured DB that is not reachable means the worker cannot + # serve requests that depend on persisted state (keys, budgets, + # spend logs). Return 503 so orchestrators take this pod out of + # rotation; "Not connected" (no DB configured at all) stays 200. + if db_health_status["status"] != "connected": + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE return { "status": "healthy", "db": db_health_status["status"], diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index 06b7d857896..2ee0588f19e 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -164,13 +164,13 @@ class _PROXY_BatchRateLimiter(CustomLogger): batch_usage: BatchFileUsage, ) -> None: """ - Check rate limits and increment counters by the batch amounts. + Atomically check + increment rate-limit counters by the batch amounts. - Raises HTTPException if any limit would be exceeded. + Raises HTTPException if any descriptor would exceed its limit; in that + case no counter is modified. Backed by `atomic_check_and_increment_by_n` + which uses a Redis Lua script when available (multi-process atomic) and + falls back to a per-process asyncio.Lock + in-memory operation. """ - from litellm.types.caching import RedisPipelineIncrementOperation - - # Create descriptors and check if batch would exceed limits descriptors = self.parallel_request_limiter._create_rate_limit_descriptors( user_api_key_dict=user_api_key_dict, data=data, @@ -179,73 +179,31 @@ class _PROXY_BatchRateLimiter(CustomLogger): model_has_failures=False, ) - # Check current usage without incrementing - rate_limit_response = await self.parallel_request_limiter.should_rate_limit( - descriptors=descriptors, - parent_otel_span=user_api_key_dict.parent_otel_span, - read_only=True, - ) + increment: Dict[Literal["requests", "tokens"], int] = { + "requests": batch_usage.request_count, + "tokens": batch_usage.total_tokens, + } + increments: List[Dict[Literal["requests", "tokens"], int]] = [ + increment for _ in descriptors + ] - # Verify batch won't exceed any limits - for status in rate_limit_response["statuses"]: - rate_limit_type = status["rate_limit_type"] - limit_remaining = status["limit_remaining"] - - required_capacity = ( - batch_usage.request_count - if rate_limit_type == "requests" - else batch_usage.total_tokens if rate_limit_type == "tokens" else 0 - ) - - if required_capacity > limit_remaining: - self._raise_rate_limit_error( - status, descriptors, batch_usage, rate_limit_type - ) - - # Build pipeline operations for batch increments - # Reuse the same keys that descriptors check - pipeline_operations: List[RedisPipelineIncrementOperation] = [] - - for descriptor in descriptors: - key = descriptor["key"] - value = descriptor["value"] - rate_limit = descriptor.get("rate_limit") - - if rate_limit is None: - continue - - # Add RPM increment if limit is set - if rate_limit.get("requests_per_unit") is not None: - rpm_key = self.parallel_request_limiter.create_rate_limit_keys( - key=key, value=value, rate_limit_type="requests" - ) - pipeline_operations.append( - RedisPipelineIncrementOperation( - key=rpm_key, - increment_value=batch_usage.request_count, - ttl=self.parallel_request_limiter.window_size, - ) - ) - - # Add TPM increment if limit is set - if rate_limit.get("tokens_per_unit") is not None: - tpm_key = self.parallel_request_limiter.create_rate_limit_keys( - key=key, value=value, rate_limit_type="tokens" - ) - pipeline_operations.append( - RedisPipelineIncrementOperation( - key=tpm_key, - increment_value=batch_usage.total_tokens, - ttl=self.parallel_request_limiter.window_size, - ) - ) - - # Execute increments - if pipeline_operations: - await self.parallel_request_limiter.async_increment_tokens_with_ttl_preservation( - pipeline_operations=pipeline_operations, + rate_limit_response = ( + await self.parallel_request_limiter.atomic_check_and_increment_by_n( + descriptors=descriptors, + increments=increments, parent_otel_span=user_api_key_dict.parent_otel_span, ) + ) + + if rate_limit_response["overall_code"] == "OVER_LIMIT": + for status in rate_limit_response["statuses"]: + if status["code"] == "OVER_LIMIT": + self._raise_rate_limit_error( + status, + descriptors, + batch_usage, + status["rate_limit_type"], + ) async def count_input_file_usage( self, diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 72483d29cdc..f7c0592992f 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -4,7 +4,7 @@ Dynamic rate limiter v3 - Saturation-aware priority-based rate limiting import os from datetime import datetime -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Callable, Dict, List, Literal, Optional, Union from fastapi import HTTPException @@ -460,92 +460,128 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if priority_descriptors: descriptors_to_check.extend(priority_descriptors) - # PHASE 1: Read-only check of ALL limits (no increments) - check_response = await self.v3_limiter.should_rate_limit( - descriptors=descriptors_to_check, + # Atomic check-and-increment for the ENFORCED descriptor set: + # - model_saturation_check is always enforced + # - priority_model is enforced only when saturation crosses threshold + # + # Backed by a Redis Lua script (multi-process atomic) with an + # asyncio.Lock + in-memory fallback for single-process deployments. + # All-or-nothing: if any enforced descriptor would exceed its limit, + # no counter is modified and the response carries "OVER_LIMIT". + enforced_descriptors: List[RateLimitDescriptor] = [model_wide_descriptor] + if priority_descriptors and should_enforce_priority: + enforced_descriptors.extend(priority_descriptors) + + per_request_increment: Dict[Literal["requests", "tokens"], int] = { + "requests": 1, + "tokens": 0, + } + atomic_response = await self.v3_limiter.atomic_check_and_increment_by_n( + descriptors=enforced_descriptors, + increments=[per_request_increment for _ in enforced_descriptors], parent_otel_span=user_api_key_dict.parent_otel_span, - read_only=True, # CRITICAL: Don't increment counters yet ) verbose_proxy_logger.debug( - f"Read-only check: {json.dumps(check_response, indent=2)}" + f"Atomic check+increment response: {json.dumps(atomic_response, indent=2)}" ) - # PHASE 2: Decide which limits to enforce - if check_response["overall_code"] == "OVER_LIMIT": - for status in check_response["statuses"]: - if status["code"] == "OVER_LIMIT": - descriptor_key = status["descriptor_key"] + if atomic_response["overall_code"] == "OVER_LIMIT": + for status in atomic_response["statuses"]: + if status["code"] != "OVER_LIMIT": + continue + descriptor_key = status["descriptor_key"] + if descriptor_key == "model_saturation_check": + raise HTTPException( + status_code=429, + detail={ + "error": f"Model capacity reached for {model}. " + f"Priority: {priority}, " + f"Rate limit type: {status['rate_limit_type']}, " + f"Remaining: {status['limit_remaining']}" + }, + headers={ + "retry-after": str(self.v3_limiter.window_size), + "rate_limit_type": str(status["rate_limit_type"]), + "x-litellm-priority": priority or "default", + }, + ) + if descriptor_key == "priority_model": + verbose_proxy_logger.debug( + f"Enforcing priority limits for {model}, saturation: {saturation:.1%}, " + f"priority: {priority}" + ) + raise HTTPException( + status_code=429, + detail={ + "error": f"Priority-based rate limit exceeded. " + f"Priority: {priority}, " + f"Rate limit type: {status['rate_limit_type']}, " + f"Remaining: {status['limit_remaining']}, " + f"Model saturation: {saturation:.1%}" + }, + headers={ + "retry-after": str(self.v3_limiter.window_size), + "rate_limit_type": str(status["rate_limit_type"]), + "x-litellm-priority": priority or "default", + "x-litellm-saturation": f"{saturation:.2%}", + }, + ) - # Model-wide limit exceeded (ALWAYS enforce) - if descriptor_key == "model_saturation_check": - raise HTTPException( - status_code=429, - detail={ - "error": f"Model capacity reached for {model}. " - f"Priority: {priority}, " - f"Rate limit type: {status['rate_limit_type']}, " - f"Remaining: {status['limit_remaining']}" - }, - headers={ - "retry-after": str(self.v3_limiter.window_size), - "rate_limit_type": str(status["rate_limit_type"]), - "x-litellm-priority": priority or "default", - }, - ) + # Fail-closed guard: overall_code says OVER_LIMIT but no status + # matched a descriptor key we know how to translate into a 429. + # Refuse the request rather than silently fall through and let an + # over-limit request proceed to the model. Without this, a future + # caller wiring an unfamiliar descriptor into enforced_descriptors + # would silently bypass the rate limit. + offending = next( + (s for s in atomic_response["statuses"] if s["code"] == "OVER_LIMIT"), + None, + ) + verbose_proxy_logger.error( + f"Dynamic rate limiter: OVER_LIMIT response with unknown " + f"descriptor_key(s) — refusing request. response={atomic_response}" + ) + raise HTTPException( + status_code=429, + detail={ + "error": "Rate limit exceeded", + "descriptor_key": ( + offending["descriptor_key"] if offending else "unknown" + ), + "rate_limit_type": ( + str(offending["rate_limit_type"]) if offending else "unknown" + ), + }, + headers={ + "retry-after": str(self.v3_limiter.window_size), + "x-litellm-priority": priority or "default", + }, + ) - # Priority limit exceeded (ONLY enforce when saturated) - elif descriptor_key == "priority_model" and should_enforce_priority: - verbose_proxy_logger.debug( - f"Enforcing priority limits for {model}, saturation: {saturation:.1%}, " - f"priority: {priority}" - ) - raise HTTPException( - status_code=429, - detail={ - "error": f"Priority-based rate limit exceeded. " - f"Priority: {priority}, " - f"Rate limit type: {status['rate_limit_type']}, " - f"Remaining: {status['limit_remaining']}, " - f"Model saturation: {saturation:.1%}" - }, - headers={ - "retry-after": str(self.v3_limiter.window_size), - "rate_limit_type": str(status["rate_limit_type"]), - "x-litellm-priority": priority or "default", - "x-litellm-saturation": f"{saturation:.2%}", - }, - ) - - # PHASE 3: Increment counters separately to avoid early-exit issues - # Model counter must ALWAYS increment, but priority counter might be over limit - # If we increment them together, v3_limiter's in-memory check will exit early - # and skip incrementing the model counter - - # Step 3a: Increment model-wide counter (always) - model_increment_response = await self.v3_limiter.should_rate_limit( - descriptors=[model_wide_descriptor], - parent_otel_span=user_api_key_dict.parent_otel_span, - read_only=False, - ) - - # Step 3b: Increment priority counter (may be over limit, but we still track it) - if priority_descriptors: - priority_increment_response = await self.v3_limiter.should_rate_limit( + # If priority is NOT enforced (saturation below threshold) but + # priority_descriptors exist, increment them for tracking only — no + # check, no rollback. This matches the prior tracking semantics. + # + # Using the non-atomic should_rate_limit (instead of + # atomic_check_and_increment_by_n) is intentional here: we don't want + # to enforce the limit, we only want to bump the counter so the + # priority allocation has accurate usage when it later becomes + # enforced. The increment-then-check semantics of should_rate_limit + # are fine because we ignore the OVER_LIMIT response. + if priority_descriptors and not should_enforce_priority: + priority_tracking_response = await self.v3_limiter.should_rate_limit( descriptors=priority_descriptors, parent_otel_span=user_api_key_dict.parent_otel_span, read_only=False, ) - - # Combine responses for post-call hook - combined_response = { - "overall_code": model_increment_response["overall_code"], - "statuses": model_increment_response["statuses"] - + priority_increment_response["statuses"], + data["litellm_proxy_rate_limit_response"] = { + "overall_code": atomic_response["overall_code"], + "statuses": atomic_response["statuses"] + + priority_tracking_response["statuses"], } - data["litellm_proxy_rate_limit_response"] = combined_response else: - data["litellm_proxy_rate_limit_response"] = model_increment_response + data["litellm_proxy_rate_limit_response"] = atomic_response async def async_pre_call_hook( self, diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index 5cdd9ddb4bd..6bb9c1f507c 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -41,6 +41,7 @@ class KeyManagementEventHooks: """ from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, + get_audit_log_changed_by, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name @@ -61,9 +62,11 @@ class KeyManagementEventHooks: request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=response.token_id or "", @@ -102,6 +105,7 @@ class KeyManagementEventHooks: """ from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, + get_audit_log_changed_by, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name @@ -117,9 +121,11 @@ class KeyManagementEventHooks: request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=data.key, @@ -140,6 +146,7 @@ class KeyManagementEventHooks: ): from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, + get_audit_log_changed_by, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name @@ -189,9 +196,11 @@ class KeyManagementEventHooks: request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.token, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=existing_key_row.token, @@ -220,6 +229,7 @@ class KeyManagementEventHooks: """ from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, + get_audit_log_changed_by, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name @@ -237,9 +247,11 @@ class KeyManagementEventHooks: request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.token, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=key.token, diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index f29bbd2d9d5..4497e64c17f 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -4,6 +4,7 @@ This is a rate limiter implementation based on a similar one by Envoy proxy. This is currently in development and not yet ready for production. """ +import asyncio import binascii import os from datetime import datetime @@ -80,6 +81,90 @@ end return results """ +CHECK_AND_INCREMENT_BY_N_SCRIPT = """ +-- Atomic check-and-increment-by-N across one or more descriptors. +-- All-or-nothing: if any descriptor would exceed its limit, no counter is +-- modified. +-- +-- Uses Redis server time (`redis.call('TIME')`) instead of a client-supplied +-- timestamp so that window resets are deterministic across replicas with +-- skewed wall-clocks. This prevents a clock-skew-induced reopening of the +-- TOCTOU window across multi-replica deployments. +-- +-- KEYS layout: pairs of (window_key, counter_key), one pair per descriptor. +-- ARGV layout: per-descriptor 4-tuple, starting at ARGV[1]: +-- ARGV[(i-1)*4 + 1] = limit +-- ARGV[(i-1)*4 + 2] = increment +-- ARGV[(i-1)*4 + 3] = ttl_seconds (counter TTL when window resets) +-- ARGV[(i-1)*4 + 4] = window_size_seconds (sliding-window length) +-- +-- Return on success: { 0, new_counter_1, new_counter_2, ... } +-- Return on over-limit: { 1, descriptor_index, current_counter, limit } +local time_reply = redis.call('TIME') +local now = tonumber(time_reply[1]) +local descriptor_count = #KEYS / 2 + +-- Pass 1: read state, validate. Abort without writing if any over limit. +local descriptor_state = {} +for i = 1, descriptor_count do + local window_key = KEYS[(i - 1) * 2 + 1] + local counter_key = KEYS[(i - 1) * 2 + 2] + local arg_base = (i - 1) * 4 + 1 + local limit = tonumber(ARGV[arg_base]) + local increment = tonumber(ARGV[arg_base + 1]) + local window_size = tonumber(ARGV[arg_base + 3]) + + local window_start = redis.call('GET', window_key) + local window_expired = (not window_start) or + ((now - tonumber(window_start)) >= window_size) + + local current_counter + if window_expired then + current_counter = 0 + else + current_counter = tonumber(redis.call('GET', counter_key) or 0) + end + + if current_counter + increment > limit then + return { 1, i, current_counter, limit } + end + + descriptor_state[i] = { window_expired, current_counter } +end + +-- Pass 2: all checks passed. Apply increments. +local results = { 0 } +for i = 1, descriptor_count do + local window_key = KEYS[(i - 1) * 2 + 1] + local counter_key = KEYS[(i - 1) * 2 + 2] + local arg_base = (i - 1) * 4 + 1 + local increment = tonumber(ARGV[arg_base + 1]) + local ttl = tonumber(ARGV[arg_base + 2]) + local window_size = tonumber(ARGV[arg_base + 3]) + + local window_expired = descriptor_state[i][1] + + if window_expired then + redis.call('SET', window_key, tostring(now)) + redis.call('SET', counter_key, increment) + redis.call('EXPIRE', window_key, window_size) + if ttl > 0 then + redis.call('EXPIRE', counter_key, ttl) + end + table.insert(results, increment) + else + local new_counter = redis.call('INCRBY', counter_key, increment) + local current_ttl = redis.call('TTL', counter_key) + if current_ttl == -1 and ttl > 0 then + redis.call('EXPIRE', counter_key, ttl) + end + table.insert(results, new_counter) + end +end + +return results +""" + TOKEN_INCREMENT_SCRIPT = """ local results = {} @@ -162,15 +247,37 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): TOKEN_INCREMENT_SCRIPT ) ) + self.check_and_increment_by_n_script = ( + self.internal_usage_cache.dual_cache.redis_cache.async_register_script( + CHECK_AND_INCREMENT_BY_N_SCRIPT + ) + ) else: self.batch_rate_limiter_script = None self.token_increment_script = None + self.check_and_increment_by_n_script = None self.window_size = int(os.getenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", 60)) # Batch rate limiter (lazy loaded) self._batch_rate_limiter: Optional[Any] = None + # Serializes multi-phase check+increment sequences (batch + dynamic + # limiters) within this process to close the TOCTOU window between + # read-only check and counter increment. Multi-replica deployments + # additionally rely on Redis Lua atomicity for cross-process safety. + # + # Coarse granularity: this single lock serializes ALL atomic check+ + # increment operations across batch and dynamic limiters on this + # instance. A slow batch input-file fetch (which happens upstream of + # the lock) does not block here, but Redis Lua latency does. If + # contention shows up under load (visible as p99 latency spikes + # correlated with batch traffic), shard to a per-descriptor-key lock + # via a `weakref.WeakValueDictionary[str, asyncio.Lock]`. Punted as a + # follow-up because Lua dominates wall-time and the lock is held for + # one round-trip. + self._check_and_increment_lock = asyncio.Lock() + def _get_batch_rate_limiter(self) -> Optional[Any]: """Get or lazy-load the batch rate limiter.""" if self._batch_rate_limiter is None: @@ -588,6 +695,281 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) return rate_limit_response + async def atomic_check_and_increment_by_n( + self, + descriptors: List[RateLimitDescriptor], + increments: List[Dict[Literal["requests", "tokens"], int]], + parent_otel_span: Optional[Span] = None, + ) -> RateLimitResponse: + """ + Atomic check-and-increment-by-N across one or more descriptors. + + All-or-nothing: if any descriptor would exceed its limit, no counter is + modified and the response carries `overall_code = "OVER_LIMIT"` with + the offending descriptor's status. Closes the TOCTOU window between + read and increment in both single-process and multi-process (Redis) + deployments. + + Args: + descriptors: rate-limit descriptors to check + increments: per-descriptor increment amounts, indexed parallel to + `descriptors`. Each entry is `{"requests": int, "tokens": int}` + — values default to 0 when a descriptor has no matching limit. + + Returns: + RateLimitResponse with one status per (descriptor, rate_limit_type) + counter, mirroring `should_rate_limit`'s shape. + """ + if len(descriptors) != len(increments): + raise ValueError( + "atomic_check_and_increment_by_n: descriptors and increments " + "must have the same length" + ) + + keys: List[str] = [] + per_counter_meta: List[Dict[str, Any]] = [] + script_args: List[Any] = [] + + for descriptor, increment_amounts in zip(descriptors, increments): + descriptor_key = descriptor["key"] + descriptor_value = descriptor["value"] + rate_limit: RateLimitDescriptorRateLimitObject = ( + descriptor.get("rate_limit") or RateLimitDescriptorRateLimitObject() + ) + window_size = rate_limit.get("window_size") or self.window_size + window_key = f"{{{descriptor_key}:{descriptor_value}}}:window" + + for rate_limit_type in ("requests", "tokens"): + rlt: Literal["requests", "tokens"] = cast( + Literal["requests", "tokens"], rate_limit_type + ) + if rlt == "requests": + limit_value = rate_limit.get("requests_per_unit") + inc_amount = int(increment_amounts.get("requests", 0) or 0) + else: + limit_value = rate_limit.get("tokens_per_unit") + inc_amount = int(increment_amounts.get("tokens", 0) or 0) + if limit_value is None or inc_amount <= 0: + continue + counter_key = self.create_rate_limit_keys( + descriptor_key, descriptor_value, rlt + ) + # Counter-key TTL and window_size are conceptually distinct + # ("how long the counter Redis key lives" vs "how long the + # sliding window is"). They happen to be equal today because + # we have no descriptor type that needs them apart, but they + # are kept as separate variables here so a future custom-TTL + # descriptor doesn't reintroduce a silent expiry bug. Both + # the Lua script and the in-memory fallback read these from + # their respective ARGV / meta slots. + ttl_seconds = int(window_size) + window_size_seconds = int(window_size) + keys.extend([window_key, counter_key]) + # Per-counter 4-tuple matches the Lua ARGV layout exactly: + # [limit, increment, ttl_seconds, window_size_seconds]. + script_args.extend( + [ + int(limit_value), + inc_amount, + ttl_seconds, + window_size_seconds, + ] + ) + per_counter_meta.append( + { + "descriptor_key": descriptor_key, + "current_limit": int(limit_value), + "rate_limit_type": rlt, + "window_key": window_key, + "counter_key": counter_key, + "increment": inc_amount, + "ttl": ttl_seconds, + "window_size": window_size_seconds, + } + ) + + if not keys: + return RateLimitResponse(overall_code="OK", statuses=[]) + + # Multi-process atomicity via Redis Lua. Single-process atomicity + # falls back to the asyncio.Lock + in-memory sliding window below. + # Note: in-memory state diverges from Redis state — if Lua fails + # mid-write, retrying via in-memory may double-count. See fallback + # warning below. + if self.check_and_increment_by_n_script is not None: + try: + raw = await self.check_and_increment_by_n_script( + keys=keys, + args=script_args, + ) + return self._build_atomic_response(raw, per_counter_meta) + except Exception as e: + # Escalated from warning to error: Lua failures (script timeout, + # Redis OOM, network partition) leave counter state ambiguous. + # The fallback path below uses LOCAL DualCache, which is a + # different store from Redis — counters here will diverge from + # Redis until that key's window expires (TTL bounds divergence). + # Operators should alert on this log line; sustained occurrences + # indicate Redis health degradation that may erode rate-limit + # accuracy. + verbose_proxy_logger.error( + f"atomic_check_and_increment_by_n: Redis Lua execution " + f"failed ({type(e).__name__}: {e}). Falling back to " + f"in-memory enforcement — counters will diverge from Redis " + f"state until window expires (window_size={self.window_size}s)." + ) + + async with self._check_and_increment_lock: + return await self._atomic_check_and_increment_in_memory( + per_counter_meta=per_counter_meta, + parent_otel_span=parent_otel_span, + ) + + def _build_atomic_response( + self, + raw: List[Any], + per_counter_meta: List[Dict[str, Any]], + ) -> RateLimitResponse: + """Convert Lua script return value to RateLimitResponse. + + Indexing invariant: `per_counter_meta` and `KEYS` are parallel-indexed + at the COUNTER level, not the descriptor level. A descriptor with both + RPM and TPM limits emits two `(window_key, counter_key)` pairs and + two meta entries — one per counter. The Lua script's loop variable + `i` therefore enumerates counters, and the over-limit return tuple + `{1, i, ...}` carries a counter index that maps directly to + `per_counter_meta[i - 1]`. Keep these arrays parallel at the counter + level when modifying this code. + """ + if not raw: + return RateLimitResponse(overall_code="OK", statuses=[]) + + status_code = int(raw[0]) + if status_code == 1: + # Over limit: { 1, counter_index (1-based), current_counter, limit } + descriptor_index = int(raw[1]) - 1 + current_counter = int(raw[2]) + limit = int(raw[3]) + meta = per_counter_meta[descriptor_index] + return RateLimitResponse( + overall_code="OVER_LIMIT", + statuses=[ + RateLimitStatus( + code="OVER_LIMIT", + current_limit=limit, + limit_remaining=max(0, limit - current_counter), + rate_limit_type=meta["rate_limit_type"], + descriptor_key=meta["descriptor_key"], + ) + ], + ) + + statuses: List[RateLimitStatus] = [] + for meta, new_counter in zip(per_counter_meta, raw[1:]): + statuses.append( + RateLimitStatus( + code="OK", + current_limit=meta["current_limit"], + limit_remaining=max(0, meta["current_limit"] - int(new_counter)), + rate_limit_type=meta["rate_limit_type"], + descriptor_key=meta["descriptor_key"], + ) + ) + return RateLimitResponse(overall_code="OK", statuses=statuses) + + async def _atomic_check_and_increment_in_memory( + self, + per_counter_meta: List[Dict[str, Any]], + parent_otel_span: Optional[Span] = None, + ) -> RateLimitResponse: + """In-memory all-or-nothing check-and-increment. Caller holds lock. + + Reads/writes the LOCAL DualCache (`local_only=True`) — note this is + a different store from Redis. When this fallback fires after a Lua + failure, in-memory counters will diverge from Redis until each key's + window expires (TTL bounds divergence). + """ + # Use a single 'now' for the duration of this critical section so all + # descriptors evaluate window expiry consistently. + now_int = int(self._get_current_time().timestamp()) + + # Pass 1: read state, validate. + descriptor_state: List[Dict[str, Any]] = [] + for meta in per_counter_meta: + window_size = meta["window_size"] + window_start = await self.internal_usage_cache.async_get_cache( + key=meta["window_key"], + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + window_expired = ( + window_start is None or (now_int - int(window_start)) >= window_size + ) + current_counter = ( + 0 + if window_expired + else int( + await self.internal_usage_cache.async_get_cache( + key=meta["counter_key"], + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + or 0 + ) + ) + if current_counter + meta["increment"] > meta["current_limit"]: + return RateLimitResponse( + overall_code="OVER_LIMIT", + statuses=[ + RateLimitStatus( + code="OVER_LIMIT", + current_limit=meta["current_limit"], + limit_remaining=max( + 0, meta["current_limit"] - current_counter + ), + rate_limit_type=meta["rate_limit_type"], + descriptor_key=meta["descriptor_key"], + ) + ], + ) + descriptor_state.append( + {"window_expired": window_expired, "current": current_counter} + ) + + # Pass 2: apply increments. + statuses: List[RateLimitStatus] = [] + for meta, state in zip(per_counter_meta, descriptor_state): + new_counter = ( + meta["increment"] + if state["window_expired"] + else state["current"] + meta["increment"] + ) + if state["window_expired"]: + await self.internal_usage_cache.async_set_cache( + key=meta["window_key"], + value=str(now_int), + ttl=meta["window_size"], + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + await self.internal_usage_cache.async_set_cache( + key=meta["counter_key"], + value=new_counter, + ttl=meta["ttl"], + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + statuses.append( + RateLimitStatus( + code="OK", + current_limit=meta["current_limit"], + limit_remaining=max(0, meta["current_limit"] - new_counter), + rate_limit_type=meta["rate_limit_type"], + descriptor_key=meta["descriptor_key"], + ) + ) + return RateLimitResponse(overall_code="OK", statuses=statuses) + def create_organization_rate_limit_descriptor( self, user_api_key_dict: UserAPIKeyAuth, requested_model: Optional[str] = None ) -> List[RateLimitDescriptor]: diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index 38623f92094..08fa8d4dfad 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -192,13 +192,19 @@ class UserManagementEventHooks: if not litellm.store_audit_logs: return + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) + await create_audit_log_for_update( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.USER_TABLE_NAME, object_id=user_id, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 5804e3f8d9f..3077efe1167 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -6,6 +6,7 @@ from collections import OrderedDict from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from fastapi import Request +from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers import litellm @@ -104,6 +105,112 @@ LITELLM_METADATA_ROUTES = ( "files", ) +_UNTRUSTED_ROOT_CONTROL_FIELDS = ( + "proxy_server_request", + "standard_logging_object", + "secret_fields", + "mock_response", + "mock_tool_calls", + "disable_global_guardrails", + "disable_global_guardrail", + "opted_out_global_guardrails", + "applied_guardrails", + "applied_policies", + "policy_sources", + "pillar_response_headers", + "_guardrail_pipelines", + "_pipeline_managed_guardrails", +) + +_UNTRUSTED_METADATA_CONTROL_FIELDS = ( + "disable_global_guardrails", + "disable_global_guardrail", + "opted_out_global_guardrails", + "pillar_response_headers", + "_pillar_response_headers_trusted", + "pillar_flagged", + "pillar_scanners", + "pillar_evidence", + "pillar_evidence_truncated", + "pillar_session_id_response", + "applied_guardrails", + "applied_policies", + "policy_sources", + "standard_logging_object", + "proxy_server_request", + "secret_fields", + "_guardrail_pipelines", + "_pipeline_managed_guardrails", +) + +_UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS = frozenset( + { + "litellm-disable-message-redaction", + } +) +_CLIENT_MOCK_CONTROL_FIELDS = frozenset({"mock_response", "mock_tool_calls"}) +_ALLOW_CLIENT_MOCK_RESPONSE_METADATA_KEY = "allow_client_mock_response" +_ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY = ( + "allow_client_message_redaction_opt_out" +) + + +def _strip_untrusted_request_header_controls( + headers: Any, + *, + allow_client_message_redaction_opt_out: bool = False, +) -> None: + if not isinstance(headers, dict): + return + + for header_name in list(headers.keys()): + if ( + isinstance(header_name, str) + and header_name.lower() in _UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS + ): + if allow_client_message_redaction_opt_out: + continue + headers.pop(header_name, None) + + +def _is_false_like(value: Any) -> bool: + if isinstance(value, bool): + return value is False + if isinstance(value, str): + return value.strip().lower() in {"false", "0", "no", "off"} + return False + + +def _key_or_team_metadata_flag_is_true( + user_api_key_dict: UserAPIKeyAuth, + metadata_key: str, +) -> bool: + for admin_metadata in (user_api_key_dict.metadata, user_api_key_dict.team_metadata): + if ( + isinstance(admin_metadata, dict) + and admin_metadata.get(metadata_key) is True + ): + return True + return False + + +def _key_or_team_allows_client_mock_response( + user_api_key_dict: UserAPIKeyAuth, +) -> bool: + return _key_or_team_metadata_flag_is_true( + user_api_key_dict=user_api_key_dict, + metadata_key=_ALLOW_CLIENT_MOCK_RESPONSE_METADATA_KEY, + ) + + +def _key_or_team_allows_client_message_redaction_opt_out( + user_api_key_dict: UserAPIKeyAuth, +) -> bool: + return _key_or_team_metadata_flag_is_true( + user_api_key_dict=user_api_key_dict, + metadata_key=_ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY, + ) + def _get_metadata_variable_name(request: Request) -> str: """ @@ -228,13 +335,25 @@ def convert_key_logging_metadata_to_callback( for var, value in data.callback_vars.items(): if team_callback_settings_obj.callback_vars is None: team_callback_settings_obj.callback_vars = {} - team_callback_settings_obj.callback_vars[var] = str( - litellm.utils.get_secret(value, default_value=value) or value - ) + team_callback_settings_obj.callback_vars[var] = str(value) return team_callback_settings_obj +def _get_validated_callback_metadata( + item: dict, *, source: str +) -> Optional[AddTeamCallback]: + try: + return AddTeamCallback(**item) + except (PydanticValidationError, ValueError) as e: + verbose_proxy_logger.warning( + "Ignoring invalid %s callback metadata: %s", + source, + _sanitize_for_log(str(e)), + ) + return None + + class KeyAndTeamLoggingSettings: """ Helper class to get the dynamic logging settings for the key and team @@ -274,8 +393,11 @@ def _get_dynamic_logging_metadata( ######################################################################################### if key_dynamic_logging_settings is not None: for item in key_dynamic_logging_settings: + callback = _get_validated_callback_metadata(item=item, source="key-level") + if callback is None: + continue callback_settings_obj = convert_key_logging_metadata_to_callback( - data=AddTeamCallback(**item), + data=callback, team_callback_settings_obj=callback_settings_obj, ) ######################################################################################### @@ -283,8 +405,11 @@ def _get_dynamic_logging_metadata( ######################################################################################### elif team_dynamic_logging_settings is not None: for item in team_dynamic_logging_settings: + callback = _get_validated_callback_metadata(item=item, source="team-level") + if callback is None: + continue callback_settings_obj = convert_key_logging_metadata_to_callback( - data=AddTeamCallback(**item), + data=callback, team_callback_settings_obj=callback_settings_obj, ) ######################################################################################### @@ -904,6 +1029,14 @@ class LiteLLMProxyRequestSetup: callback_vars_dict.pop("team_id", None) callback_vars_dict.pop("success_callback", None) callback_vars_dict.pop("failure_callback", None) + callback_vars_dict = { + key: ( + litellm.utils.get_secret(value, default_value=value) or value + if isinstance(value, str) + else value + ) + for key, value in callback_vars_dict.items() + } return TeamCallbackMetadata( success_callback=team_config.get("success_callback", None), @@ -962,11 +1095,15 @@ async def add_litellm_data_to_request( # noqa: PLR0915 # Strip internal-only keys from user input before the proxy sets its own. # These keys are injected by the proxy itself below — user-supplied values # must not be trusted. - for _internal_key in ( - "proxy_server_request", - "standard_logging_object", - "secret_fields", - ): + _allow_client_mock_response = _key_or_team_allows_client_mock_response( + user_api_key_dict + ) + _allow_client_message_redaction_opt_out = ( + _key_or_team_allows_client_message_redaction_opt_out(user_api_key_dict) + ) + for _internal_key in _UNTRUSTED_ROOT_CONTROL_FIELDS: + if _allow_client_mock_response and _internal_key in _CLIENT_MOCK_CONTROL_FIELDS: + continue data.pop(_internal_key, None) # Strip spoofable auth metadata from user-supplied metadata dict _user_metadata = data.get("metadata") @@ -1007,6 +1144,17 @@ async def add_litellm_data_to_request( # noqa: PLR0915 forward_llm_provider_auth_headers=forward_llm_auth, authenticated_with_header=authenticated_with_header, ) + _strip_untrusted_request_header_controls( + _headers, + allow_client_message_redaction_opt_out=_allow_client_message_redaction_opt_out, + ) + if ( + not _allow_client_message_redaction_opt_out + and litellm.turn_off_message_logging is True + and "turn_off_message_logging" in data + and _is_false_like(data["turn_off_message_logging"]) + ): + data.pop("turn_off_message_logging", None) verbose_proxy_logger.debug(f"Request Headers: {_headers}") verbose_proxy_logger.debug(f"Raw Headers: {_raw_headers}") @@ -1144,8 +1292,18 @@ async def add_litellm_data_to_request( # noqa: PLR0915 for _meta_key in ("metadata", "litellm_metadata"): _user_meta = data.get(_meta_key) if isinstance(_user_meta, dict): - _user_meta.pop("_pipeline_managed_guardrails", None) - for _k in [k for k in _user_meta if k.startswith("user_api_key_")]: + _strip_untrusted_request_header_controls( + _user_meta.get("headers"), + allow_client_message_redaction_opt_out=( + _allow_client_message_redaction_opt_out + ), + ) + for _k in [ + k + for k in _user_meta + if k.startswith("user_api_key_") + or k in _UNTRUSTED_METADATA_CONTROL_FIELDS + ]: _user_meta.pop(_k, None) # Strip caller-supplied routing/budget tags unless the admin has opted diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 61996850bb9..62a770f46ae 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -247,13 +247,12 @@ async def _patch_key_caches_add_access_group( ) -> None: """Patch cached key objects to include access_group_id.""" for token in key_tokens: - cached_key = await user_api_key_cache.async_get_cache(key=token) + cached_key = await user_api_key_cache.async_get_cache( + key=token, + model_type=UserAPIKeyAuth, + ) if cached_key is None: continue - if isinstance(cached_key, dict): - cached_key = UserAPIKeyAuth(**cached_key) - if not isinstance(cached_key, UserAPIKeyAuth): - continue if cached_key.access_group_ids is None: cached_key.access_group_ids = [access_group_id] elif access_group_id not in cached_key.access_group_ids: @@ -278,12 +277,11 @@ async def _patch_key_caches_remove_access_group( ) -> None: """Patch cached key objects to remove access_group_id.""" for token in key_tokens: - cached_key = await user_api_key_cache.async_get_cache(key=token) - if cached_key is None: - continue - if isinstance(cached_key, dict): - cached_key = UserAPIKeyAuth(**cached_key) - if isinstance(cached_key, UserAPIKeyAuth) and cached_key.access_group_ids: + cached_key = await user_api_key_cache.async_get_cache( + key=token, + model_type=UserAPIKeyAuth, + ) + if cached_key is not None and cached_key.access_group_ids: cached_key.access_group_ids = [ ag for ag in cached_key.access_group_ids if ag != access_group_id ] diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index c6d37ace4fe..921d24da043 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -2069,6 +2069,9 @@ async def delete_user( litellm_proxy_admin_name, prisma_client, ) + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) @@ -2162,9 +2165,11 @@ async def delete_user( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.USER_TABLE_NAME, object_id=user_id, diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a424f1558fb..a01f5e63211 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -27,7 +27,7 @@ from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, s import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid -from litellm.caching import DualCache +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.constants import ( LENGTH_OF_LITELLM_GENERATED_KEY, LITELLM_PROXY_ADMIN_NAME, @@ -37,6 +37,7 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._experimental.mcp_server.db import ( rotate_mcp_server_credentials_master_key, + rotate_mcp_user_credentials_master_key, ) from litellm.proxy._types import * from litellm.proxy._types import LiteLLM_VerificationToken @@ -65,6 +66,7 @@ from litellm.proxy.management_helpers.object_permission_utils import ( attach_object_permission_to_dict, handle_update_object_permission_common, validate_key_mcp_servers_against_team, + validate_key_search_tools_against_team, ) from litellm.proxy.management_helpers.team_member_permission_checks import ( TeamMemberPermissionChecks, @@ -768,6 +770,10 @@ async def _common_key_generation_helper( # noqa: PLR0915 object_permission=data_json.get("object_permission"), team_obj=team_table, ) + await validate_key_search_tools_against_team( + object_permission=data_json.get("object_permission"), + team_obj=team_table, + ) data_json = await _set_object_permission( data_json=data_json, @@ -1053,7 +1059,7 @@ async def _check_project_key_limits( project_id: str, data: Union[GenerateKeyRequest, UpdateKeyRequest], prisma_client: PrismaClient, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, ) -> None: """ Validate that key's models and budget respect its project's limits. @@ -1828,7 +1834,7 @@ async def _process_single_key_update( user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str], prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: Any, llm_router: Optional[Router], user_custom_key_update: Optional[Callable] = None, @@ -2010,6 +2016,10 @@ async def _validate_mcp_servers_for_key_update( object_permission=object_permission_dict, team_obj=effective_team_obj, ) + await validate_key_search_tools_against_team( + object_permission=object_permission_dict, + team_obj=effective_team_obj, + ) async def _validate_update_key_data( @@ -3288,7 +3298,7 @@ async def _team_key_deletion_check( user_api_key_dict: UserAPIKeyAuth, key_info: LiteLLM_VerificationToken, prisma_client: PrismaClient, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, ): is_team_key = _is_team_key(data=key_info) @@ -3331,7 +3341,7 @@ async def _team_key_deletion_check( async def can_modify_verification_token( key_info: LiteLLM_VerificationToken, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, ) -> bool: @@ -3405,7 +3415,7 @@ async def can_modify_verification_token( async def delete_verification_tokens( tokens: List, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str] = None, ) -> Tuple[Optional[Dict], List[LiteLLM_VerificationToken]]: @@ -3595,7 +3605,7 @@ async def _persist_deleted_verification_tokens( async def delete_key_aliases( key_aliases: List[str], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str] = None, @@ -3709,6 +3719,17 @@ async def _rotate_master_key( # noqa: PLR0915 "Failed to rotate MCP server credentials: %s", str(e) ) + # 4b. process MCP user-scoped credentials table (BYOK + OAuth2 tokens) + try: + await rotate_mcp_user_credentials_master_key( + prisma_client=prisma_client, + new_master_key=new_master_key, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to rotate MCP user credentials: %s", str(e) + ) + # 5. process credentials table try: credentials = await prisma_client.db.litellm_credentialstable.find_many() @@ -3841,7 +3862,7 @@ async def _execute_virtual_key_regeneration( data: Optional[RegenerateKeyRequest], user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str], - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, ) -> GenerateKeyResponse: """Generate new token, update DB, invalidate cache, and return response.""" @@ -4131,7 +4152,7 @@ async def _check_proxy_or_team_admin_for_key( key_in_db: LiteLLM_VerificationToken, user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, ) -> None: if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value: return @@ -5152,7 +5173,7 @@ async def _check_key_admin_access( user_api_key_dict: UserAPIKeyAuth, hashed_token: str, prisma_client: Any, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, route: str, ) -> None: """ @@ -5245,6 +5266,9 @@ async def block_key( proxy_logging_obj, user_api_key_cache, ) + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) if prisma_client is None: raise Exception("{}".format(CommonProxyErrors.db_not_connected_error.value)) @@ -5288,9 +5312,11 @@ async def block_key( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=hashed_token, @@ -5354,6 +5380,9 @@ async def unblock_key( proxy_logging_obj, user_api_key_cache, ) + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) if prisma_client is None: raise Exception("{}".format(CommonProxyErrors.db_not_connected_error.value)) @@ -5397,9 +5426,11 @@ async def unblock_key( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=hashed_token, @@ -5580,7 +5611,6 @@ async def test_key_logging( "content": "Hello, this is a test from litellm /key/health. No LLM API call was made for this", } ], - "mock_response": "test response", } data = await add_litellm_data_to_request( data=data, @@ -5589,6 +5619,7 @@ async def test_key_logging( general_settings=general_settings, request=request, ) + data["mock_response"] = "test response" await litellm.acompletion( **data ) # make mock completion call to trigger key based callbacks diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index cc84c36f9cf..729493e1df8 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -56,6 +56,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy.management_helpers.audit_logs import get_audit_log_changed_by router = APIRouter(prefix="/v1/mcp", tags=["mcp"]) @@ -2232,7 +2233,12 @@ if MCP_AVAILABLE: detail={"error": "Only proxy admins can create MCP toolsets."}, ) touched_by = ( - litellm_changed_by or user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME + get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME, + ) + or LITELLM_PROXY_ADMIN_NAME ) try: result = await create_mcp_toolset(prisma_client, payload, touched_by) @@ -2323,7 +2329,12 @@ if MCP_AVAILABLE: detail={"error": "Only proxy admins can update MCP toolsets."}, ) touched_by = ( - litellm_changed_by or user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME + get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME, + ) + or LITELLM_PROXY_ADMIN_NAME ) try: result = await update_mcp_toolset(prisma_client, payload, touched_by) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 4eec7c6b7c0..a6c0c7dcc0e 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -4,15 +4,22 @@ Endpoints to control callbacks per team Use this when each team should control its own callbacks """ +import asyncio +import copy import json import traceback -from typing import List, Optional +from datetime import datetime, timezone +from typing import Any, List, Optional from fastapi import APIRouter, Depends, Header, HTTPException, Request, status +import litellm from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid from litellm.proxy._types import ( AddTeamCallback, + LiteLLM_AuditLogs, + LitellmTableNames, ProxyErrorTypes, ProxyException, TeamCallbackMetadata, @@ -24,6 +31,106 @@ from litellm.proxy.management_helpers.utils import management_endpoint_wrapper router = APIRouter() +_CALLBACK_VARS_REDACTED = "***REDACTED***" + + +def _redact_callback_secrets(metadata: Any) -> Any: + """Strip secret values out of a team-metadata snapshot before audit logging. + + Both ``team_metadata["logging"]`` (list of ``AddTeamCallback`` dicts) and + ``team_metadata["callback_settings"]["callback_vars"]`` carry provider + credentials such as ``langfuse_secret_key``, ``langsmith_api_key``, and + ``gcs_path_service_account``. Persisting them verbatim into + ``LiteLLM_AuditLogs`` would let anyone with read access to the audit + table harvest team callback credentials, so we replace each value with + a fixed marker. The keys themselves are kept so the audit reader can + still see *which* fields changed. + """ + if not isinstance(metadata, dict): + return metadata + redacted = copy.deepcopy(metadata) + logging_entries = redacted.get("logging") + if isinstance(logging_entries, list): + for entry in logging_entries: + if isinstance(entry, dict) and isinstance(entry.get("callback_vars"), dict): + entry["callback_vars"] = { + k: _CALLBACK_VARS_REDACTED for k in entry["callback_vars"] + } + callback_settings = redacted.get("callback_settings") + if isinstance(callback_settings, dict) and isinstance( + callback_settings.get("callback_vars"), dict + ): + callback_settings["callback_vars"] = { + k: _CALLBACK_VARS_REDACTED for k in callback_settings["callback_vars"] + } + return redacted + + +def _log_audit_task_exception(task: "asyncio.Task[None]") -> None: + """Surface a fire-and-forget audit-log task failure. + + ``asyncio.create_task`` swallows exceptions silently — if the audit + write fails (transient DB error etc.) we'd otherwise lose the row + without any signal. Log at warning level so the operator sees there's + a gap in the audit trail. + """ + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + verbose_proxy_logger.warning("Failed to write team-callback audit log: %s", exc) + + +async def _emit_team_callback_audit_log( + *, + team_id: str, + before_metadata: Any, + after_metadata: Any, + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: Optional[str], +) -> None: + """Emit an audit-log row for a team-callback mutation. + + Mirrors the ``store_audit_logs``-gated pattern used in + ``team_endpoints.py``: the call is async-fire-and-forget and is a no-op + when audit logging is not enabled on the proxy. Captured under + ``LitellmTableNames.TEAM_TABLE_NAME`` so the row co-locates with other + team mutations in the audit table. + + Callback secrets are redacted before serialization so the audit table + cannot itself become a credential-harvest sink. + """ + if litellm.store_audit_logs is not True: + return + + from litellm.proxy.management_helpers.audit_logs import ( + create_audit_log_for_update, + ) + from litellm.proxy.proxy_server import litellm_proxy_admin_name + + redacted_before = _redact_callback_secrets(before_metadata) + redacted_after = _redact_callback_secrets(after_metadata) + + task = asyncio.create_task( + create_audit_log_for_update( + request_data=LiteLLM_AuditLogs( + id=str(uuid.uuid4()), + updated_at=datetime.now(timezone.utc), + changed_by=litellm_changed_by + or user_api_key_dict.user_id + or litellm_proxy_admin_name, + changed_by_api_key=user_api_key_dict.api_key, + table_name=LitellmTableNames.TEAM_TABLE_NAME, + object_id=team_id, + action="updated", + updated_values=json.dumps({"metadata": redacted_after}, default=str), + before_value=json.dumps({"metadata": redacted_before}, default=str), + ) + ) + ) + task.add_done_callback(_log_audit_task_exception) + + @router.post( "/team/{team_id:path}/callback", tags=["team management"], @@ -123,6 +230,7 @@ async def add_team_callbacks( param="callback_name", ) + before_metadata = copy.deepcopy(team_metadata) team_callback_settings.append(data.model_dump()) team_metadata["logging"] = team_callback_settings @@ -132,6 +240,14 @@ async def add_team_callbacks( where={"team_id": team_id}, data={"metadata": team_metadata_json} # type: ignore ) + await _emit_team_callback_audit_log( + team_id=team_id, + before_metadata=before_metadata, + after_metadata=team_metadata, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + return { "status": "success", "data": new_team_row, @@ -165,6 +281,10 @@ async def disable_team_logging( http_request: Request, team_id: str, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + litellm_changed_by: Optional[str] = Header( + None, + description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + ), ): """ Disable all logging callbacks for a team @@ -198,6 +318,7 @@ async def disable_team_logging( # Update team metadata to disable logging team_metadata = _existing_team.metadata + before_metadata = copy.deepcopy(team_metadata) team_callback_settings = team_metadata.get("callback_settings", {}) team_callback_settings_obj = TeamCallbackMetadata(**team_callback_settings) @@ -222,6 +343,17 @@ async def disable_team_logging( }, ) + # Disabling a team's logging callbacks is itself a logging-control + # action — emit an audit-log row so the action remains traceable + # even though the team's own observability is now off. + await _emit_team_callback_audit_log( + team_id=team_id, + before_metadata=before_metadata, + after_metadata=team_metadata, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + return { "status": "success", "message": f"Logging disabled for team {team_id}", diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index d53c1e71300..466ce47a6fd 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -906,6 +906,9 @@ async def new_team( # noqa: PLR0915 prisma_client, user_api_key_cache, ) + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) @@ -1174,9 +1177,11 @@ async def new_team( # noqa: PLR0915 request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.TEAM_TABLE_NAME, object_id=data.team_id, @@ -1214,7 +1219,10 @@ async def _create_team_update_audit_log( user_api_key_dict: User API key authentication details litellm_proxy_admin_name: Name of the proxy admin """ - from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update + from litellm.proxy.management_helpers.audit_logs import ( + create_audit_log_for_update, + get_audit_log_changed_by, + ) _before_value = existing_team_row.json(exclude_none=True) _before_value = json.dumps(_before_value, default=str) @@ -1225,9 +1233,11 @@ async def _create_team_update_audit_log( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.TEAM_TABLE_NAME, object_id=team_id, @@ -2003,21 +2013,34 @@ def team_member_add_duplication_check( async def _validate_team_member_add_permissions( user_api_key_dict: UserAPIKeyAuth, complete_team_data: LiteLLM_TeamTable, + data: TeamMemberAddRequest, ) -> None: - """Validate if user has permission to add members to the team.""" + """Validate if user has permission to add members to the team. + + Standard users can self-join an *available team*, but the bypass + must not be allowed to escalate them to ``role=admin`` or to add + other users into the team. When access is granted via the + available-team bypass we therefore enforce that every member in + the request matches the caller's own ``user_id`` and is being + added with ``role="user"``. + """ if ( - hasattr(user_api_key_dict, "user_role") - and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value - and not _is_user_team_admin( - user_api_key_dict=user_api_key_dict, team_obj=complete_team_data - ) - and not await _is_user_org_admin_for_team( - user_api_key_dict=user_api_key_dict, team_obj=complete_team_data - ) - and not _is_available_team( - team_id=complete_team_data.team_id, - user_api_key_dict=user_api_key_dict, - ) + getattr(user_api_key_dict, "user_role", None) + == LitellmUserRoles.PROXY_ADMIN.value + ): + return + if _is_user_team_admin( + user_api_key_dict=user_api_key_dict, team_obj=complete_team_data + ): + return + if await _is_user_org_admin_for_team( + user_api_key_dict=user_api_key_dict, team_obj=complete_team_data + ): + return + + if not _is_available_team( + team_id=complete_team_data.team_id, + user_api_key_dict=user_api_key_dict, ): raise HTTPException( status_code=403, @@ -2029,6 +2052,34 @@ async def _validate_team_member_add_permissions( }, ) + # Available-team self-join: caller may add only themselves, only as a + # standard user. Enforce that here so the bypass cannot be used as a + # privilege-escalation or cross-user-injection primitive. + members = data.member if isinstance(data.member, list) else [data.member] + caller_user_id = getattr(user_api_key_dict, "user_id", None) + for member in members: + if getattr(member, "role", "user") != "user": + raise HTTPException( + status_code=403, + detail={ + "error": ( + "Available-team self-join cannot assign 'admin' role. " + "Only proxy/team/org admins can add admins to a team." + ) + }, + ) + member_user_id = getattr(member, "user_id", None) + if not caller_user_id or not member_user_id or member_user_id != caller_user_id: + raise HTTPException( + status_code=403, + detail={ + "error": ( + "Available-team self-join can only add the caller " + "(user_id must match the authenticated user's user_id)." + ) + }, + ) + async def _process_team_members( data: TeamMemberAddRequest, @@ -2049,8 +2100,11 @@ async def _process_team_members( # Resolve allowed_models: explicit request value, or fall back to team's default_team_member_models member_allowed_models = data.allowed_models - if member_allowed_models is None and complete_team_data.default_team_member_models: - member_allowed_models = complete_team_data.default_team_member_models + team_default_member_models = getattr( + complete_team_data, "default_team_member_models", None + ) + if member_allowed_models is None and team_default_member_models: + member_allowed_models = team_default_member_models if isinstance(data.member, Member): try: @@ -2381,6 +2435,7 @@ async def team_member_add( await _validate_team_member_add_permissions( user_api_key_dict=user_api_key_dict, complete_team_data=complete_team_data, + data=data, ) # Validate and populate user_email/user_id for members before processing @@ -2992,6 +3047,9 @@ async def delete_team( litellm_proxy_admin_name, prisma_client, ) + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + ) if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) @@ -3051,9 +3109,11 @@ async def delete_team( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.TEAM_TABLE_NAME, object_id=team_id, @@ -4696,6 +4756,8 @@ async def update_team_member_permissions( complete_team_data = LiteLLM_TeamTable(**existing_team_row.model_dump()) + # Available-team self-join must NOT grant write access to team-wide + # permission policies; only proxy/team/org admins can update them. if ( hasattr(user_api_key_dict, "user_role") and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value @@ -4705,16 +4767,12 @@ async def update_team_member_permissions( and not await _is_user_org_admin_for_team( user_api_key_dict=user_api_key_dict, team_obj=complete_team_data ) - and not _is_available_team( - team_id=complete_team_data.team_id, - user_api_key_dict=user_api_key_dict, - ) ): raise HTTPException( status_code=403, detail={ "error": "Call not allowed. User not proxy admin OR team admin. route={}, team_id={}".format( - "/team/member_add", + "/team/permissions_update", complete_team_data.team_id, ) }, diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 46e7963da7c..9dfc67370fe 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -13,7 +13,9 @@ import base64 import hashlib import inspect import os +import re import secrets +from html import escape from copy import deepcopy from typing import ( TYPE_CHECKING, @@ -27,20 +29,23 @@ from typing import ( Union, cast, ) -from urllib.parse import urlencode, urlparse +from urllib.parse import parse_qs, urlencode, urlparse if TYPE_CHECKING: import httpx import jwt -from fastapi import APIRouter, Depends, HTTPException, Request, status +from fastapi import APIRouter, Depends, Header, HTTPException, Request, status from fastapi.responses import RedirectResponse import litellm +from litellm.caching.dual_cache import DualCache from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid -from litellm.caching import DualCache from litellm.constants import ( + CLI_SSO_SESSION_CACHE_KEY_PREFIX, + CLI_SSO_SESSION_TTL_SECONDS, + LITELLM_CLI_SOURCE_IDENTIFIER, LITELLM_UI_SESSION_DURATION, MAX_SPENDLOG_ROWS_TO_QUERY, MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE, @@ -70,7 +75,11 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken, get_user_object -from litellm.proxy.auth.auth_utils import _has_user_setup_sso +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.auth.auth_utils import ( + _get_request_ip_address, + _has_user_setup_sso, +) from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.admin_ui_utils import ( @@ -123,6 +132,250 @@ router = APIRouter() # Metadata fields (token_type, expires_in, scope) are intentionally kept so # response convertors see the same fields in the PKCE path as in the non-PKCE path. _OAUTH_TOKEN_FIELDS = frozenset({"access_token", "id_token", "refresh_token"}) +_CLI_SSO_FLOW_CACHE_KEY_PREFIX = f"{CLI_SSO_SESSION_CACHE_KEY_PREFIX}:flow" +_CLI_SSO_START_RATE_LIMIT_CACHE_KEY_PREFIX = ( + f"{_CLI_SSO_FLOW_CACHE_KEY_PREFIX}:start_rate_limit" +) +_CLI_SSO_START_RATE_LIMIT_WINDOW_SECONDS = 60 +_CLI_SSO_START_RATE_LIMIT_MAX_ATTEMPTS = 30 +_CLI_SSO_USER_CODE_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" +_CLI_SSO_LOGIN_ID_RE = re.compile(r"^cli-[A-Za-z0-9_-]{12,124}$") + + +def _hash_cli_sso_secret(secret: str) -> str: + return hashlib.sha256(secret.encode("utf-8")).hexdigest() + + +def _normalize_cli_sso_user_code(user_code: str) -> str: + return "".join(ch for ch in user_code.upper() if ch.isalnum()) + + +def _generate_cli_sso_user_code() -> str: + user_code = "".join(secrets.choice(_CLI_SSO_USER_CODE_ALPHABET) for _ in range(8)) + return f"{user_code[:4]}-{user_code[4:]}" + + +def _get_cli_sso_flow_cache_key(login_id: str) -> str: + return f"{_CLI_SSO_FLOW_CACHE_KEY_PREFIX}:{login_id}" + + +def _is_valid_cli_sso_login_id(login_id: Optional[str]) -> bool: + return isinstance(login_id, str) and bool(_CLI_SSO_LOGIN_ID_RE.fullmatch(login_id)) + + +def _get_cli_sso_start_rate_limit_cache_key( + request: Request, use_x_forwarded_for: Optional[bool] = False +) -> str: + client_ip = ( + _get_request_ip_address( + request=request, use_x_forwarded_for=use_x_forwarded_for + ) + or "unknown" + ) + client_ip_hash = _hash_cli_sso_secret(client_ip) + return f"{_CLI_SSO_START_RATE_LIMIT_CACHE_KEY_PREFIX}:{client_ip_hash}" + + +def _check_cli_sso_start_rate_limit( + request: Request, + cache: DualCache, + use_x_forwarded_for: Optional[bool] = False, +) -> None: + rate_limit_cache_key = _get_cli_sso_start_rate_limit_cache_key( + request=request, use_x_forwarded_for=use_x_forwarded_for + ) + current_attempts = cache.increment_cache( + key=rate_limit_cache_key, + value=1, + ttl=_CLI_SSO_START_RATE_LIMIT_WINDOW_SECONDS, + ) + if current_attempts > _CLI_SSO_START_RATE_LIMIT_MAX_ATTEMPTS: + raise HTTPException( + status_code=429, + detail="Too many CLI login attempts. Try again later.", + ) + + +def _get_cli_sso_flow_or_raise(login_id: Optional[str], cache: DualCache) -> dict: + if not _is_valid_cli_sso_login_id(login_id): + raise HTTPException(status_code=400, detail="Invalid CLI login session") + + cache_key = _get_cli_sso_flow_cache_key(cast(str, login_id)) + flow = cache.get_cache(key=cache_key) + if not isinstance(flow, dict) or "poll_secret_hash" not in flow: + raise HTTPException(status_code=400, detail="Invalid CLI login session") + return flow + + +def _set_cli_sso_flow(login_id: str, cache: DualCache, flow: dict) -> None: + cache.set_cache( + key=_get_cli_sso_flow_cache_key(login_id), + value=flow, + ttl=CLI_SSO_SESSION_TTL_SECONDS, + ) + + +def _verify_cli_sso_poll_secret(flow: dict, poll_secret: Optional[str]) -> bool: + expected_poll_secret_hash = flow.get("poll_secret_hash") + if not isinstance(expected_poll_secret_hash, str) or not isinstance( + poll_secret, str + ): + return False + supplied_poll_secret_hash = _hash_cli_sso_secret(poll_secret) + return secrets.compare_digest(supplied_poll_secret_hash, expected_poll_secret_hash) + + +def _render_cli_sso_verification_page( + verify_url: str, browser_complete_token: str +) -> str: + escaped_verify_url = escape(verify_url, quote=True) + escaped_browser_complete_token = escape(browser_complete_token, quote=True) + return f""" + + + + LiteLLM CLI Login + + + +
+

Complete CLI Login

+

Enter the verification code shown in your terminal to finish this login.

+
+ + + + +
+
+ + + """ + + +@router.post("/sso/cli/start", tags=["experimental"], include_in_schema=False) +async def cli_sso_start(request: Request): + from litellm.proxy.proxy_server import general_settings, user_api_key_cache + + _check_cli_sso_start_rate_limit( + request=request, + cache=user_api_key_cache, + use_x_forwarded_for=bool( + (general_settings or {}).get("use_x_forwarded_for", False) + ), + ) + + login_id = f"cli-{secrets.token_urlsafe(24)}" + poll_secret = secrets.token_urlsafe(32) + user_code = _generate_cli_sso_user_code() + + flow = { + "poll_secret_hash": _hash_cli_sso_secret(poll_secret), + "user_code_hash": _hash_cli_sso_secret(_normalize_cli_sso_user_code(user_code)), + "sso_complete": False, + "user_code_verified": False, + "session_data": None, + } + _set_cli_sso_flow(login_id=login_id, cache=user_api_key_cache, flow=flow) + + return { + "login_id": login_id, + "poll_secret": poll_secret, + "user_code": user_code, + "expires_in": CLI_SSO_SESSION_TTL_SECONDS, + } + + +@router.post( + "/sso/cli/complete/{login_id}", tags=["experimental"], include_in_schema=False +) +async def cli_sso_complete(request: Request, login_id: str): + from fastapi.responses import HTMLResponse + + from litellm.proxy.common_utils.html_forms.cli_sso_success import ( + render_cli_sso_success_page, + ) + from litellm.proxy.proxy_server import user_api_key_cache + + flow = _get_cli_sso_flow_or_raise(login_id=login_id, cache=user_api_key_cache) + if not flow.get("sso_complete") or not flow.get("session_data"): + raise HTTPException(status_code=400, detail="CLI login is not ready") + + body = (await request.body()).decode("utf-8") + form_values = parse_qs(body) + supplied_user_code = (form_values.get("user_code") or [""])[0] + supplied_browser_complete_token = ( + form_values.get("browser_complete_token") or [""] + )[0] + supplied_user_code_hash = _hash_cli_sso_secret( + _normalize_cli_sso_user_code(supplied_user_code) + ) + supplied_browser_complete_token_hash = _hash_cli_sso_secret( + supplied_browser_complete_token + ) + + expected_user_code_hash = flow.get("user_code_hash") + if not isinstance(expected_user_code_hash, str) or not secrets.compare_digest( + supplied_user_code_hash, expected_user_code_hash + ): + raise HTTPException(status_code=400, detail="Invalid verification code") + + expected_browser_complete_token_hash = flow.get("browser_complete_token_hash") + if not isinstance( + expected_browser_complete_token_hash, str + ) or not secrets.compare_digest( + supplied_browser_complete_token_hash, expected_browser_complete_token_hash + ): + raise HTTPException(status_code=400, detail="Invalid verification code") + + flow["user_code_verified"] = True + _set_cli_sso_flow(login_id=login_id, cache=user_api_key_cache, flow=flow) + + html_content = render_cli_sso_success_page() + return HTMLResponse(content=html_content, status_code=200) def normalize_email(email: Optional[str]) -> Optional[str]: @@ -333,6 +586,7 @@ async def google_login( from litellm.proxy.proxy_server import ( premium_user, prisma_client, + user_api_key_cache, user_custom_ui_sso_sign_in_handler, ) @@ -382,14 +636,15 @@ async def google_login( redirect_url = SSOAuthenticationHandler.get_redirect_url_for_sso( request=request, sso_callback_route="sso/callback", - existing_key=existing_key, ) - # Store CLI key in state for OAuth flow + if source == LITELLM_CLI_SOURCE_IDENTIFIER: + _get_cli_sso_flow_or_raise(login_id=key, cache=user_api_key_cache) + + # Store CLI login handle in state for OAuth flow cli_state: Optional[str] = SSOAuthenticationHandler._get_cli_state( source=source, key=key, - existing_key=existing_key, ) # check if user defined a custom auth sso sign in handler, if yes, use it @@ -1050,7 +1305,7 @@ async def get_existing_user_info_from_db( user_id: Optional[str], user_email: Optional[str], prisma_client: PrismaClient, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, ) -> Optional[LiteLLM_UserTable]: try: @@ -1074,7 +1329,7 @@ async def get_existing_user_info_from_db( async def get_user_info_from_db( result: Union[CustomOpenID, OpenID, dict], prisma_client: PrismaClient, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, user_email: Optional[str], user_defined_values: Optional[SSOUserDefinedValues], @@ -1194,7 +1449,7 @@ async def _sync_user_role_from_jwt_role_map( received_response: Optional[dict], user_info: Optional[Union[LiteLLM_UserTable, NewUserResponse]], prisma_client: PrismaClient, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, user_defined_values: Optional[SSOUserDefinedValues], ) -> None: """ @@ -1233,11 +1488,8 @@ async def _sync_user_role_from_jwt_role_map( user_info.user_role = mapped_role.value await user_api_key_cache.async_set_cache( key=user_info.user_id, - value=( - user_info.model_dump() - if hasattr(user_info, "model_dump") - else dict(user_info) - ), + value=user_info, + model_type=LiteLLM_UserTable, ) @@ -1392,18 +1644,12 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa: ) if state and state.startswith(f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:"): - # Extract the key ID and existing_key from the state - # State format: {PREFIX}:{key}:{existing_key} or {PREFIX}:{key} - state_parts = state.split(":", 2) # Split into max 3 parts + # State format: {PREFIX}:{login_id} + state_parts = state.split(":", 1) key_id = state_parts[1] if len(state_parts) > 1 else None - existing_key = state_parts[2] if len(state_parts) > 2 else None - verbose_proxy_logger.info( - f"CLI SSO callback detected for key: {key_id}, existing_key: {existing_key}" - ) - return await cli_sso_callback( - request=request, key=key_id, existing_key=existing_key, result=result - ) + verbose_proxy_logger.info("CLI SSO callback detected") + return await cli_sso_callback(request=request, key=key_id, result=result) # Control-plane cross-origin: read return_to from cookie. # Starlette's cookie_parser already handles RFC 2109 unquoting. @@ -1424,13 +1670,10 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa: async def cli_sso_callback( request: Request, key: Optional[str] = None, - existing_key: Optional[str] = None, result: Optional[Union[OpenID, dict]] = None, ): """CLI SSO callback - stores session info for JWT generation on polling""" - verbose_proxy_logger.info( - f"CLI SSO callback for key: {key}, existing_key: {existing_key}" - ) + verbose_proxy_logger.info("CLI SSO callback") from litellm.proxy.proxy_server import ( prisma_client, @@ -1438,11 +1681,7 @@ async def cli_sso_callback( user_api_key_cache, ) - if not key or not key.startswith("sk-"): - raise HTTPException( - status_code=400, - detail="Invalid key parameter. Must be a valid key ID starting with 'sk-'", - ) + flow = _get_cli_sso_flow_or_raise(login_id=key, cache=user_api_key_cache) if prisma_client is None: raise HTTPException( @@ -1480,9 +1719,6 @@ async def cli_sso_callback( status_code=500, detail="Failed to retrieve user information from SSO" ) - # Store session info in cache (10 min TTL) - from litellm.constants import CLI_SSO_SESSION_CACHE_KEY_PREFIX - # Get all teams from user_info - CLI will let user select which one teams: List[str] = [] if hasattr(user_info, "teams") and user_info.teams: @@ -1523,21 +1759,25 @@ async def cli_sso_callback( "team_details": team_details, } - cache_key = f"{CLI_SSO_SESSION_CACHE_KEY_PREFIX}:{key}" - user_api_key_cache.set_cache(key=cache_key, value=session_data, ttl=600) + flow["session_data"] = session_data + flow["sso_complete"] = True + browser_complete_token = secrets.token_urlsafe(32) + flow["browser_complete_token_hash"] = _hash_cli_sso_secret( + browser_complete_token + ) + _set_cli_sso_flow(login_id=cast(str, key), cache=user_api_key_cache, flow=flow) verbose_proxy_logger.info( f"Stored CLI SSO session for user: {user_info.user_id}, teams: {teams}, num_teams: {len(teams)}" ) - # Return success page from fastapi.responses import HTMLResponse - from litellm.proxy.common_utils.html_forms.cli_sso_success import ( - render_cli_sso_success_page, + verify_url = str(request.url_for("cli_sso_complete", login_id=key)) + html_content = _render_cli_sso_verification_page( + verify_url=verify_url, + browser_complete_token=browser_complete_token, ) - - html_content = render_cli_sso_success_page() return HTMLResponse(content=html_content, status_code=200) except Exception as e: @@ -1548,7 +1788,11 @@ async def cli_sso_callback( @router.get("/sso/cli/poll/{key_id}", tags=["experimental"], include_in_schema=False) -async def cli_poll_key(key_id: str, team_id: Optional[str] = None): +async def cli_poll_key( + key_id: str, + team_id: Optional[str] = None, + x_litellm_cli_poll_secret: Optional[str] = Header(default=None), +): """ CLI polling endpoint - retrieves session from cache and generates JWT. @@ -1557,22 +1801,25 @@ async def cli_poll_key(key_id: str, team_id: Optional[str] = None): 2. Second poll (with team_id): Generates JWT with selected team and deletes session Args: - key_id: The session key ID + key_id: The CLI login session ID team_id: Optional team ID to assign to the JWT. If provided, must be one of user's teams. """ - from litellm.constants import CLI_SSO_SESSION_CACHE_KEY_PREFIX from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken from litellm.proxy.proxy_server import user_api_key_cache - if not key_id.startswith("sk-"): - raise HTTPException(status_code=400, detail="Invalid key ID format") - try: - # Look up session in cache - cache_key = f"{CLI_SSO_SESSION_CACHE_KEY_PREFIX}:{key_id}" - session_data = user_api_key_cache.get_cache(key=cache_key) + flow = _get_cli_sso_flow_or_raise(login_id=key_id, cache=user_api_key_cache) + if not _verify_cli_sso_poll_secret( + flow=flow, poll_secret=x_litellm_cli_poll_secret + ): + raise HTTPException(status_code=403, detail="Invalid CLI polling secret") - if session_data: + if not flow.get("sso_complete") or not flow.get("user_code_verified"): + return {"status": "pending"} + + session_data = flow.get("session_data") + + if isinstance(session_data, dict): user_teams = session_data.get("teams", []) user_team_details = session_data.get("team_details") user_id = session_data["user_id"] @@ -1632,7 +1879,7 @@ async def cli_poll_key(key_id: str, team_id: Optional[str] = None): ) # Delete cache entry (single-use) - user_api_key_cache.delete_cache(key=cache_key) + user_api_key_cache.delete_cache(key=_get_cli_sso_flow_cache_key(key_id)) verbose_proxy_logger.info( f"CLI JWT generated for user: {user_id}, team: {team_id}" @@ -1650,6 +1897,8 @@ async def cli_poll_key(key_id: str, team_id: Optional[str] = None): else: return {"status": "pending"} + except HTTPException: + raise except Exception as e: verbose_proxy_logger.error(f"Error polling for CLI JWT: {e}") raise HTTPException( @@ -2393,20 +2642,15 @@ class SSOAuthenticationHandler: This is used to authenticate through the CLI login flow. - The state parameter format is: {PREFIX}:{key}:{existing_key} - - If existing_key is provided, it's included in the state + The state parameter format is: {PREFIX}:{login_id} - The state parameter is used to pass data through the OAuth flow without changing the callback URL """ from litellm.constants import ( LITELLM_CLI_SESSION_TOKEN_PREFIX, - LITELLM_CLI_SOURCE_IDENTIFIER, ) if source == LITELLM_CLI_SOURCE_IDENTIFIER and key: - if existing_key: - return f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:{key}:{existing_key}" - else: - return f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:{key}" + return f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:{key}" else: return None diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index 7599e11bdef..d3b225e6e4d 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -21,6 +21,28 @@ from litellm.proxy._types import ( from litellm.types.utils import StandardAuditLogPayload _audit_log_callback_cache: Dict[str, CustomLogger] = {} +ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY = "allow_litellm_changed_by_header" + + +def _allows_litellm_changed_by_header(user_api_key_dict: UserAPIKeyAuth) -> bool: + for admin_metadata in (user_api_key_dict.metadata, user_api_key_dict.team_metadata): + if ( + isinstance(admin_metadata, dict) + and admin_metadata.get(ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY) is True + ): + return True + return False + + +def get_audit_log_changed_by( + *, + litellm_changed_by: Optional[str], + user_api_key_dict: UserAPIKeyAuth, + litellm_proxy_admin_name: Optional[str], +) -> Optional[str]: + if litellm_changed_by and _allows_litellm_changed_by_header(user_api_key_dict): + return litellm_changed_by + return user_api_key_dict.user_id or litellm_proxy_admin_name def _resolve_audit_log_callback(name: str) -> Optional[CustomLogger]: @@ -143,8 +165,10 @@ async def create_object_audit_log( if _store_audit_logs is not True: return - _changed_by = ( - litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name + _changed_by = get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, ) await create_audit_log_for_update( diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index 410f636693f..eb90d1b5ca7 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -335,8 +335,9 @@ async def validate_key_mcp_servers_against_team( disallowed_servers = requested_servers - all_allowed_servers if disallowed_servers: if team_obj is not None: + team_id = team_obj.team_id detail = ( - f"Key requests MCP servers not allowed by team '{team_obj.team_id}': " + f"Key requests MCP servers not allowed by team '{team_id}': " f"{sorted(disallowed_servers)}. " f"Team allows: {sorted(team_allowed_servers)}. " f"Global (allow_all_keys) servers: {sorted(allow_all_keys_servers)}." @@ -365,8 +366,9 @@ async def validate_key_mcp_servers_against_team( disallowed_groups = requested_access_groups - team_access_groups if disallowed_groups: if team_obj is not None: + team_id = team_obj.team_id detail = ( - f"Key requests MCP access groups not allowed by team '{team_obj.team_id}': " + f"Key requests MCP access groups not allowed by team '{team_id}': " f"{sorted(disallowed_groups)}. " f"Team allows: {sorted(team_access_groups)}." ) @@ -390,13 +392,60 @@ async def validate_key_mcp_servers_against_team( if team_mcp_toolsets: disallowed_toolsets = requested_toolsets - set(team_mcp_toolsets) if disallowed_toolsets: + team_id = team_obj.team_id raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail={ "error": ( - f"Key requests MCP toolsets not allowed by team '{team_obj.team_id}': " + f"Key requests MCP toolsets not allowed by team '{team_id}': " f"{sorted(disallowed_toolsets)}. " f"Team allows: {sorted(team_mcp_toolsets)}." ) }, ) + + +def _extract_requested_search_tools(object_permission: Optional[dict]) -> List[str]: + """Return search_tool_name values from a key's object_permission dict.""" + if not object_permission or not isinstance(object_permission, dict): + return [] + raw = object_permission.get("search_tools") + if not isinstance(raw, list): + return [] + return [str(x) for x in raw if x] + + +async def validate_key_search_tools_against_team( + object_permission: Optional[dict], + team_obj: Optional["LiteLLM_TeamTableCachedObj"], +) -> None: + """ + Validate key object_permission.search_tools is a subset of the team's allowlist. + + Empty team allowlist means no restriction at team layer (skip). + """ + requested = _extract_requested_search_tools(object_permission) + if not requested: + return + + team_tools: List[str] = [] + if team_obj is not None and team_obj.object_permission is not None: + st = team_obj.object_permission.search_tools + if st: + team_tools = list(st) + + if not team_tools: + return + + disallowed = set(requested) - set(team_tools) + if disallowed: + team_id = team_obj.team_id if team_obj is not None else "unknown" + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": ( + f"Key requests search tools not allowed by team '{team_id}': " + f"{sorted(disallowed)}. Team allows: {sorted(team_tools)}." + ) + }, + ) diff --git a/litellm/proxy/management_helpers/team_member_permission_checks.py b/litellm/proxy/management_helpers/team_member_permission_checks.py index e035168ca00..50339210a6e 100644 --- a/litellm/proxy/management_helpers/team_member_permission_checks.py +++ b/litellm/proxy/management_helpers/team_member_permission_checks.py @@ -1,6 +1,5 @@ from typing import List, Optional -from litellm.caching import DualCache from litellm.proxy._types import ( KeyManagementRoutes, LiteLLM_TeamTableCachedObj, @@ -12,6 +11,7 @@ from litellm.proxy._types import ( ProxyException, UserAPIKeyAuth, ) +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.auth.auth_checks import get_team_object from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.utils import PrismaClient @@ -65,7 +65,7 @@ class TeamMemberPermissionChecks: user_api_key_dict: UserAPIKeyAuth, route: KeyManagementRoutes, prisma_client: PrismaClient, - user_api_key_cache: DualCache, + user_api_key_cache: UserApiKeyCache, existing_key_row: LiteLLM_VerificationToken, ): """ diff --git a/litellm/proxy/middleware/prometheus_auth_middleware.py b/litellm/proxy/middleware/prometheus_auth_middleware.py index 6bdff59da52..3b30fd3d63c 100644 --- a/litellm/proxy/middleware/prometheus_auth_middleware.py +++ b/litellm/proxy/middleware/prometheus_auth_middleware.py @@ -3,6 +3,7 @@ Prometheus Auth Middleware - Pure ASGI implementation """ import json +from typing import Any, List, MutableMapping from fastapi import Request from starlette.types import ASGIApp, Receive, Scope, Send @@ -40,8 +41,17 @@ class PrometheusAuthMiddleware: # Only run auth if configured to do so if litellm.require_auth_for_metrics_endpoint is True: - # Construct Request only when auth is actually needed - request = Request(scope, receive) + # user_api_key_auth reads the request body, which consumes ASGI `receive`. + # Buffer those messages and replay them for the inner app; otherwise a + # successful auth would forward an exhausted receive and /metrics hangs. + buffered_messages: List[MutableMapping[str, Any]] = [] + + async def receive_for_auth() -> MutableMapping[str, Any]: + message = await receive() + buffered_messages.append(message) + return message + + request = Request(scope, receive_for_auth) api_key = request.headers.get(_AUTHORIZATION_HEADER) or "" try: @@ -70,5 +80,18 @@ class PrometheusAuthMiddleware: ) return + replay_idx = 0 + + async def receive_replay() -> MutableMapping[str, Any]: + nonlocal replay_idx + if replay_idx < len(buffered_messages): + msg = buffered_messages[replay_idx] + replay_idx += 1 + return msg + return await receive() + + await self.app(scope, receive_replay, send) + return + # Pass through to the inner application await self.app(scope, receive, send) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 216eb61a9d1..c42faa59cf0 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -549,10 +549,16 @@ class AnthropicPassthroughLoggingHandler: # Create a mock user API key dict for the managed object storage from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + _request_metadata = (kwargs.get("litellm_params", {}) or {}).get( + "metadata", {} + ) or {} + user_api_key_dict = UserAPIKeyAuth( - user_id=kwargs.get("user_id", "default-user"), + user_id=_request_metadata.get( + "user_api_key_user_id", "default-user" + ), api_key="", - team_id=None, + team_id=_request_metadata.get("user_api_key_team_id"), team_alias=None, user_role=LitellmUserRoles.CUSTOMER, # Use proper enum value user_email=None, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py index a8c5562d4d6..6277f6b4a75 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py @@ -1,6 +1,7 @@ import asyncio import json import time +import urllib.parse from datetime import datetime from typing import Literal, Optional from urllib.parse import urlparse @@ -203,8 +204,16 @@ class AssemblyAIPassthroughLoggingHandler: ) if _api_key is None: raise ValueError("AssemblyAI API key not found") + if ( + any(c in transcript_id for c in ("/", "\\", "#", "?")) + or ".." in transcript_id + ): + raise ValueError( + f"Invalid transcript_id {transcript_id!r}: contains disallowed characters" + ) + safe_transcript_id = urllib.parse.quote(transcript_id, safe="") try: - url = f"{_base_url}/v2/transcript/{transcript_id}" + url = f"{_base_url}/v2/transcript/{safe_transcript_id}" headers = { "Authorization": f"Bearer {_api_key}", "Content-Type": "application/json", diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 86dd23e12ca..6a138532617 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -849,10 +849,16 @@ class VertexPassthroughLoggingHandler: # Create a mock user API key dict for the managed object storage from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + _request_metadata = (kwargs.get("litellm_params", {}) or {}).get( + "metadata", {} + ) or {} + user_api_key_dict = UserAPIKeyAuth( - user_id=kwargs.get("user_id", "default-user"), + user_id=_request_metadata.get( + "user_api_key_user_id", "default-user" + ), api_key="", - team_id=None, + team_id=_request_metadata.get("user_api_key_team_id"), team_alias=None, user_role=LitellmUserRoles.CUSTOMER, # Use proper enum value user_email=None, diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 77eb3a5ee0c..714b5f3c7b6 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -41,7 +41,6 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.passthrough import BasePassthroughUtils from litellm.proxy._types import ( - CommonProxyErrors, ConfigFieldInfo, ConfigFieldUpdate, LiteLLMRoutes, @@ -2325,12 +2324,14 @@ async def _register_pass_through_endpoint( dependencies = None if auth is not None and str(auth).lower() == "true": - if premium_user is not True: - raise ValueError( - "Error Setting Authentication on Pass Through Endpoint: {}".format( - CommonProxyErrors.not_premium_user.value - ) - ) + # Authentication on a pass-through endpoint used to be enterprise- + # only — which left the OSS tier with no safe configuration: the + # default was ``auth=False`` (unauthenticated forwarder) and the + # safe ``auth=True`` raised at startup unless the operator had a + # license. The default is now ``True`` (safe-by-default), and + # turning it on no longer requires a license: an unauthenticated + # forwarder is a deployment choice the operator should be allowed + # to make explicitly, but the safe option must always be free. dependencies = [Depends(user_api_key_auth)] if path not in LiteLLMRoutes.openai_routes.value: LiteLLMRoutes.openai_routes.value.append(path) diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 302d7e76edf..cbfcd34c438 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -36,21 +36,16 @@ class PassThroughStreamingHandler: passthrough_success_handler_obj: PassThroughEndpointLogging, url_route: str, ): - """ - - Yields chunks from the response - - Collect non-empty chunks for post-processing (logging) - - Inject cost into chunks if include_cost_in_streaming_usage is enabled - """ - try: - raw_bytes: List[bytes] = [] - # Extract model name for cost injection - model_name = PassThroughStreamingHandler._extract_model_for_cost_injection( - request_body=request_body, - url_route=url_route, - endpoint_type=endpoint_type, - litellm_logging_obj=litellm_logging_obj, - ) + raw_bytes: List[bytes] = [] + logging_scheduled = False + model_name = PassThroughStreamingHandler._extract_model_for_cost_injection( + request_body=request_body, + url_route=url_route, + endpoint_type=endpoint_type, + litellm_logging_obj=litellm_logging_obj, + ) + try: async for chunk in response.aiter_bytes(): raw_bytes.append(chunk) if ( @@ -58,7 +53,6 @@ class PassThroughStreamingHandler: and model_name ): if endpoint_type == EndpointType.VERTEX_AI: - # Only handle streamRawPredict (uses Anthropic format) if "streamRawPredict" in url_route or "rawPredict" in url_route: modified_chunk = ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection( chunk, model_name @@ -73,25 +67,32 @@ class PassThroughStreamingHandler: chunk = modified_chunk yield chunk - - # After all chunks are processed, handle post-processing - end_time = datetime.now() - - asyncio.create_task( - PassThroughStreamingHandler._route_streaming_logging_to_handler( - litellm_logging_obj=litellm_logging_obj, - passthrough_success_handler_obj=passthrough_success_handler_obj, - url_route=url_route, - request_body=request_body or {}, - endpoint_type=endpoint_type, - start_time=start_time, - raw_bytes=raw_bytes, - end_time=end_time, - ) - ) except Exception as e: verbose_proxy_logger.error(f"Error in chunk_processor: {str(e)}") raise + finally: + # GeneratorExit (raised on client disconnect) is not caught by + # `except Exception`; the finally block ensures partial usage + # still gets logged for spend tracking. See LIT-2642. + if not logging_scheduled and raw_bytes: + logging_scheduled = True + try: + asyncio.create_task( + PassThroughStreamingHandler._route_streaming_logging_to_handler( + litellm_logging_obj=litellm_logging_obj, + passthrough_success_handler_obj=passthrough_success_handler_obj, + url_route=url_route, + request_body=request_body or {}, + endpoint_type=endpoint_type, + start_time=start_time, + raw_bytes=raw_bytes, + end_time=datetime.now(), + ) + ) + except Exception as e: + verbose_proxy_logger.error( + f"Error scheduling chunk_processor logging: {str(e)}" + ) @staticmethod async def _route_streaming_logging_to_handler( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5f5566c7c58..b053ee1492a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -78,8 +78,11 @@ from litellm.proxy._types import ( InvitationNew, InvitationUpdate, Litellm_EntityType, + LiteLLM_EndUserTable, LiteLLM_JWTAuth, + LiteLLM_TagTable, LiteLLM_TeamTable, + LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, LitellmUserRoles, PassThroughGenericEndpoint, @@ -91,8 +94,10 @@ from litellm.proxy._types import ( TeamDefaultSettings, TokenCountRequest, TransformRequestBody, + UI_TEAM_ID, UserAPIKeyAuth, ) +from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec from litellm.proxy.common_utils.callback_utils import ( normalize_callback_names, process_callback, @@ -205,6 +210,7 @@ from litellm import Router from litellm._logging import verbose_proxy_logger, verbose_router_logger from litellm.caching.caching import DualCache, RedisCache from litellm.caching.redis_cluster_cache import RedisClusterCache +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.constants import ( _REALTIME_BODY_CACHE_SIZE, APSCHEDULER_COALESCE, @@ -235,37 +241,11 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.vertex_llm_base import VertexBase -from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( - router as mcp_byok_oauth_router, -) -from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - router as mcp_discoverable_endpoints_router, -) -from litellm.proxy._experimental.mcp_server.rest_endpoints import ( - router as mcp_rest_endpoints_router, -) -from litellm.proxy._experimental.mcp_server.server import app as mcp_app -from litellm.proxy._experimental.mcp_server.tool_registry import ( - global_mcp_tool_registry, -) from litellm.proxy._types import * -from litellm.proxy.agent_endpoints.a2a_endpoints import router as a2a_router -from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry -from litellm.proxy.agent_endpoints.endpoints import router as agent_endpoints_router -from litellm.proxy.agent_endpoints.model_list_helpers import ( - append_agents_to_model_group, - append_agents_to_model_info, -) +from litellm.proxy._lazy_features import attach_lazy_features from litellm.proxy.analytics_endpoints.analytics_endpoints import ( router as analytics_router, ) -from litellm.proxy.anthropic_endpoints.claude_code_endpoints import ( - claude_code_marketplace_router, -) -from litellm.proxy.anthropic_endpoints.endpoints import router as anthropic_router -from litellm.proxy.anthropic_endpoints.skills_endpoints import ( - router as anthropic_skills_router, -) from litellm.proxy.auth.auth_checks import ( ExperimentalUIJWTToken, get_team_object, @@ -328,7 +308,6 @@ from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router from litellm.proxy.fine_tuning_endpoints.endpoints import set_fine_tuning_config from litellm.proxy.google_endpoints.endpoints import router as google_router -from litellm.proxy.guardrails.guardrail_endpoints import router as guardrails_router from litellm.proxy.guardrails.init_guardrails import ( init_guardrails_v2, initialize_guardrails, @@ -344,9 +323,6 @@ from litellm.proxy.hooks.prompt_injection_detection import ( from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger from litellm.proxy.image_endpoints.endpoints import router as image_router from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request -from litellm.proxy.management_endpoints.access_group_endpoints import ( - router as access_group_router, -) from litellm.proxy.management_endpoints.budget_management_endpoints import ( router as budget_management_router, ) @@ -361,12 +337,6 @@ from litellm.proxy.management_endpoints.common_utils import ( _user_has_admin_view, admin_can_invite_user, ) -from litellm.proxy.management_endpoints.compliance_endpoints import ( - router as compliance_router, -) -from litellm.proxy.management_endpoints.config_override_endpoints import ( - router as config_override_router, -) from litellm.proxy.management_endpoints.cost_tracking_settings import ( router as cost_tracking_settings_router, ) @@ -380,9 +350,6 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import ( router as internal_user_router, ) from litellm.proxy.management_endpoints.internal_user_endpoints import user_update -from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import ( - router as jwt_key_mapping_router, -) from litellm.proxy.management_endpoints.key_management_endpoints import ( delete_verification_tokens, duration_in_seconds, @@ -391,9 +358,6 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( from litellm.proxy.management_endpoints.key_management_endpoints import ( router as key_management_router, ) -from litellm.proxy.management_endpoints.mcp_management_endpoints import ( - router as mcp_management_router, -) from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( router as model_access_group_management_router, ) @@ -408,11 +372,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( from litellm.proxy.management_endpoints.organization_endpoints import ( router as organization_router, ) -from litellm.proxy.management_endpoints.policy_endpoints import router as policy_router from litellm.proxy.management_endpoints.router_settings_endpoints import ( router as router_settings_router, ) -from litellm.proxy.management_endpoints.scim.scim_v2 import scim_router from litellm.proxy.management_endpoints.tag_management_endpoints import ( router as tag_management_router, ) @@ -424,9 +386,6 @@ from litellm.proxy.management_endpoints.team_endpoints import ( update_team, validate_membership, ) -from litellm.proxy.management_endpoints.tool_management_endpoints import ( - router as tool_management_router, -) from litellm.proxy.management_endpoints.workflow_management_endpoints import ( router as workflow_management_router, ) @@ -435,7 +394,6 @@ from litellm.proxy.management_endpoints.ui_sso import ( get_disabled_non_admin_personal_key_creation, ) from litellm.proxy.management_endpoints.ui_sso import router as ui_sso_router -from litellm.proxy.management_endpoints.usage_endpoints import router as usage_ai_router from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import ( router as user_agent_analytics_router, ) @@ -445,7 +403,6 @@ from litellm.proxy.middleware.in_flight_requests_middleware import ( ) from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router -from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_router from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, ) @@ -465,27 +422,16 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( router as pass_through_router, ) -from litellm.proxy.policy_engine.policy_endpoints import router as policy_crud_router -from litellm.proxy.policy_engine.policy_resolve_endpoints import ( - router as policy_resolve_router, -) -from litellm.proxy.prompts.prompt_endpoints import router as prompts_router from litellm.proxy.public_endpoints import router as public_endpoints_router from litellm.proxy.rag_endpoints.endpoints import router as rag_router -from litellm.proxy.realtime_endpoints.endpoints import router as webrtc_router from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router from litellm.proxy.response_api_endpoints.endpoints import router as response_router from litellm.proxy.route_llm_request import route_request from litellm.proxy.search_endpoints.endpoints import router as search_router -from litellm.proxy.search_endpoints.search_tool_management import ( - router as search_tool_management_router, -) -from litellm.proxy.spend_tracking.cloudzero_endpoints import router as cloudzero_router from litellm.proxy.spend_tracking.spend_management_endpoints import ( router as spend_management_router, ) from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload -from litellm.proxy.spend_tracking.vantage_endpoints import router as vantage_router from litellm.proxy.types_utils.utils import get_instance_fn from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( router as ui_crud_endpoints_router, @@ -515,16 +461,6 @@ from litellm.proxy.utils import ( prefetch_config_params, update_spend, ) -from litellm.proxy.vector_store_endpoints.endpoints import router as vector_store_router -from litellm.proxy.vector_store_endpoints.management_endpoints import ( - router as vector_store_management_router, -) -from litellm.proxy.vector_store_files_endpoints.endpoints import ( - router as vector_store_files_router, -) -from litellm.proxy.vertex_ai_endpoints.langfuse_endpoints import ( - router as langfuse_router, -) from litellm.proxy.video_endpoints.endpoints import router as video_router from litellm.router import ( AssistantsTypedDict, @@ -1104,6 +1040,11 @@ def get_openapi_schema(): openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema) + # Stub unloaded lazy features so they appear as Swagger sections. + from litellm.proxy._lazy_features import inject_lazy_stubs + + openapi_schema = inject_lazy_stubs(openapi_schema) + # Fix Swagger UI execute path error when server_root_path is set if server_root_path: openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}] @@ -1130,6 +1071,11 @@ def custom_openapi(): openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema) + # Stub unloaded lazy features so they appear as Swagger sections. + from litellm.proxy._lazy_features import inject_lazy_stubs + + openapi_schema = inject_lazy_stubs(openapi_schema) + # Fix Swagger UI execute path error when server_root_path is set if server_root_path: openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}] @@ -1557,14 +1503,78 @@ def mount_swagger_ui(): app.mount("/swagger", StaticFiles(directory=swagger_directory), name="swagger") + # On dropdown expand: one-time fetch to the prefix (triggers lazy load), + # then spec re-download so real routes replace the stub. Raw JS (no + #