From a1514efa210c60c00809b21d2906503b0c452cc8 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Mon, 27 Jul 2026 17:07:58 +0200 Subject: [PATCH 001/175] fix(vector_stores): S3 Vectors search router bypass + rag query config drop + UI error swallow --- .../azure_ai/vector_stores/transformation.py | 2 + .../base_llm/vector_store/transformation.py | 4 + .../bedrock/vector_stores/transformation.py | 2 + litellm/llms/custom_httpx/llm_http_handler.py | 7 + .../gemini/vector_stores/transformation.py | 2 + .../milvus/vector_stores/transformation.py | 2 + .../openai/vector_stores/transformation.py | 2 + .../pg_vector/vector_stores/transformation.py | 2 + .../ragflow/vector_stores/transformation.py | 2 + .../vector_stores/transformation.py | 32 ++- .../vector_stores/rag_api/transformation.py | 2 + .../search_api/transformation.py | 2 + litellm/proxy/rag_endpoints/endpoints.py | 14 ++ litellm/rag/main.py | 14 +- litellm/router.py | 8 + litellm/vector_stores/main.py | 9 +- .../test_s3_vectors_transformation.py | 189 +++++++++++++++++- .../proxy/rag_endpoints/test_rag_endpoints.py | 103 ++++++++++ tests/test_litellm/rag/test_main.py | 90 +++++++++ tests/test_litellm/test_router.py | 55 +++++ tests/test_litellm/vector_stores/test_main.py | 77 +++++++ .../_components/VectorStoreTester.test.tsx | 25 ++- .../_components/VectorStoreTester.tsx | 8 +- .../src/components/networking.tsx | 2 +- 24 files changed, 628 insertions(+), 27 deletions(-) create mode 100644 tests/test_litellm/vector_stores/test_main.py diff --git a/litellm/llms/azure_ai/vector_stores/transformation.py b/litellm/llms/azure_ai/vector_stores/transformation.py index da6a4a93cd8..bd3eaeee989 100644 --- a/litellm/llms/azure_ai/vector_stores/transformation.py +++ b/litellm/llms/azure_ai/vector_stores/transformation.py @@ -19,6 +19,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -92,6 +93,7 @@ class AzureAIVectorStoreConfig(BaseVectorStoreConfig, BaseAzureLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Azure AI Search API diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index b222e3dd160..9a0e401b527 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -16,6 +16,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router from ..chat.transformation import BaseLLMException as _BaseLLMException @@ -56,6 +57,7 @@ class BaseVectorStoreConfig: litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: pass @@ -68,6 +70,7 @@ class BaseVectorStoreConfig: litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """ Optional async version of transform_search_vector_store_request. @@ -83,6 +86,7 @@ class BaseVectorStoreConfig: litellm_logging_obj=litellm_logging_obj, litellm_params=litellm_params, extra_body=extra_body, + router=router, ) @abstractmethod diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index c1b124caec1..7a6a0eb6d84 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -27,6 +27,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -196,6 +197,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: if isinstance(query, list): query = " ".join(query) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ec1301e5923..ec701fbe87e 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -167,6 +167,7 @@ if TYPE_CHECKING: AnthropicMessagesStreamingResponse, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig + from litellm.router import Router from litellm.types.llms.openai_evals import ( CancelEvalResponse, CancelRunResponse, @@ -9409,6 +9410,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + router: Optional["Router"] = None, ) -> VectorStoreSearchResponse: if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( @@ -9443,6 +9445,7 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, + router=router, ) else: ( @@ -9456,6 +9459,7 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, + router=router, ) all_optional_params: Dict[str, Any] = dict(litellm_params) all_optional_params.update(vector_store_search_optional_params or {}) @@ -9507,6 +9511,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + router: Optional["Router"] = None, ) -> Union[VectorStoreSearchResponse, Coroutine[Any, Any, VectorStoreSearchResponse]]: if _is_async: return self.async_vector_store_search_handler( @@ -9521,6 +9526,7 @@ class BaseLLMHTTPHandler: extra_body=extra_body, timeout=timeout, client=client, + router=router, ) if client is None or not isinstance(client, HTTPHandler): @@ -9551,6 +9557,7 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, + router=router, ) all_optional_params: Dict[str, Any] = dict(litellm_params) diff --git a/litellm/llms/gemini/vector_stores/transformation.py b/litellm/llms/gemini/vector_stores/transformation.py index f98cb0e5b0c..5aba5752a44 100644 --- a/litellm/llms/gemini/vector_stores/transformation.py +++ b/litellm/llms/gemini/vector_stores/transformation.py @@ -31,6 +31,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -111,6 +112,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """ Transform search request to Gemini's generateContent format. diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py index a53075ba1d6..589063cd188 100644 --- a/litellm/llms/milvus/vector_stores/transformation.py +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -19,6 +19,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -123,6 +124,7 @@ class MilvusVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Azure AI Search API diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py index 6ccf8e271e5..9ab1568a375 100644 --- a/litellm/llms/openai/vector_stores/transformation.py +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -21,6 +21,7 @@ from litellm.utils import add_openai_metadata if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -99,6 +100,7 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: encoded_vector_store_id = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url = f"{api_base}/{encoded_vector_store_id}/search" diff --git a/litellm/llms/pg_vector/vector_stores/transformation.py b/litellm/llms/pg_vector/vector_stores/transformation.py index b58b6e7f498..116f79c834f 100644 --- a/litellm/llms/pg_vector/vector_stores/transformation.py +++ b/litellm/llms/pg_vector/vector_stores/transformation.py @@ -8,6 +8,7 @@ from litellm.types.vector_stores import VectorStoreSearchOptionalRequestParams if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -80,6 +81,7 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: encoded_vector_store_id = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url = f"{api_base}/{encoded_vector_store_id}/search" diff --git a/litellm/llms/ragflow/vector_stores/transformation.py b/litellm/llms/ragflow/vector_stores/transformation.py index d8bdd981425..332ed7f0c6b 100644 --- a/litellm/llms/ragflow/vector_stores/transformation.py +++ b/litellm/llms/ragflow/vector_stores/transformation.py @@ -17,6 +17,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -92,6 +93,7 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """RAGFlow vector stores are management-only, search is not supported.""" raise NotImplementedError("RAGFlow vector stores support dataset management only, not search/retrieval") diff --git a/litellm/llms/s3_vectors/vector_stores/transformation.py b/litellm/llms/s3_vectors/vector_stores/transformation.py index b31e6f4511a..a999db21dbe 100644 --- a/litellm/llms/s3_vectors/vector_stores/transformation.py +++ b/litellm/llms/s3_vectors/vector_stores/transformation.py @@ -1,8 +1,8 @@ -import re from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import httpx +from litellm.caching._embedding_router import resolve_embedding_router from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.types.router import GenericLiteLLMParams @@ -18,6 +18,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -58,13 +59,18 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): return headers def get_complete_url(self, api_base: Optional[str], litellm_params: dict) -> str: - aws_region_name = litellm_params.get("aws_region_name") - if not aws_region_name: - raise ValueError("aws_region_name is required for S3 Vectors") - if not re.match(r"^[a-z][a-z0-9-]*$", aws_region_name): - raise ValueError("Invalid aws_region_name format") + # Resolve region the same way the ingestion path does: + # dynamic param -> AWS_REGION_NAME -> AWS_REGION -> default (us-west-2) + aws_region_name = self.get_aws_region_name_for_non_llm_api_calls(litellm_params.get("aws_region_name")) return f"https://s3vectors.{aws_region_name}.api.aws" + def _resolve_query_embedding_router(self, embedding_model: str, router: Optional["Router"]) -> Optional["Router"]: + """Return the router iff it serves ``embedding_model`` as a deployment.""" + if router is None: + return None + model_list = [dict(m) for m in (router.get_model_list() or [])] + return resolve_embedding_router(embedding_model=embedding_model, llm_router=router, llm_model_list=model_list) + def transform_search_vector_store_request( self, vector_store_id: str, @@ -74,6 +80,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """Sync version - generates embedding synchronously.""" # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name @@ -99,10 +106,14 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): # Generate embedding for the query embedding_model = litellm_params.get("embedding_model", "text-embedding-3-small") + embedding_router = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router) import litellm as litellm_module - embedding_response = litellm_module.embedding(model=embedding_model, input=[query]) + if embedding_router is not None: + embedding_response = embedding_router.embedding(model=embedding_model, input=[query]) + else: + embedding_response = litellm_module.embedding(model=embedding_model, input=[query]) query_embedding = embedding_response.data[0]["embedding"] url = f"{api_base}/QueryVectors" @@ -128,6 +139,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """Async version - generates embedding asynchronously.""" # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name @@ -153,10 +165,14 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): # Generate embedding for the query asynchronously embedding_model = litellm_params.get("embedding_model", "text-embedding-3-small") + embedding_router = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router) import litellm as litellm_module - embedding_response = await litellm_module.aembedding(model=embedding_model, input=[query]) + if embedding_router is not None: + embedding_response = await embedding_router.aembedding(model=embedding_model, input=[query]) + else: + embedding_response = await litellm_module.aembedding(model=embedding_model, input=[query]) query_embedding = embedding_response.data[0]["embedding"] url = f"{api_base}/QueryVectors" diff --git a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py index 47a81fc07bf..93ad40616b5 100644 --- a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py @@ -19,6 +19,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -97,6 +98,7 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Vertex AI RAG API diff --git a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py index 958839d4a48..f6f9e34dc75 100644 --- a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py @@ -23,6 +23,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -197,6 +198,7 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform a search request for the Vertex AI Search (Discovery Engine) API. diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index 27ffc49901b..0d93f20373c 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -26,6 +26,9 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _safe_get_request_headers, get_form_data, ) +from litellm.proxy.vector_store_endpoints.endpoints import ( + _update_request_data_with_litellm_managed_vector_store_registry, +) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, ) @@ -652,6 +655,17 @@ async def rag_query( user_api_key_dict=user_api_key_dict, ) + # Merge litellm-managed vector store params (provider, region, embedding + # model, credentials, ...) from the registry — same source the direct + # /vector_stores/{id}/search endpoint uses. User-supplied + # retrieval_config keys win on conflict. + store_data = await _update_request_data_with_litellm_managed_vector_store_registry( + data={}, + vector_store_id=retrieval_config["vector_store_id"], + user_api_key_dict=user_api_key_dict, + ) + retrieval_config = {**store_data, **retrieval_config} + # Add litellm data request_data: Dict[str, Any] = {} request_data = await add_litellm_data_to_request( diff --git a/litellm/rag/main.py b/litellm/rag/main.py index 29891ccfd24..2329a820f1f 100644 --- a/litellm/rag/main.py +++ b/litellm/rag/main.py @@ -59,6 +59,14 @@ INGESTION_REGISTRY: Dict[str, Type[BaseRAGIngestion]] = { "vertex_ai": VertexAIRAGIngestion, } +# retrieval_config keys consumed by the query pipeline itself; everything else is +# forwarded to vector_stores.asearch as provider-specific params (e.g. +# aws_region_name, embedding_model, vector_bucket_name for S3 Vectors). +# `filters`/`retrieval_filter` are reserved for the explicit filter param. +_CONSUMED_RETRIEVAL_CONFIG_KEYS = frozenset( + {"vector_store_id", "custom_llm_provider", "top_k", "filters", "retrieval_filter"} +) + def get_ingestion_class(provider: str) -> Type[BaseRAGIngestion]: """ @@ -233,13 +241,17 @@ async def _execute_query_pipeline( raise ValueError("No query found in messages for RAG query") # 2. Search vector store + # Forward provider-specific retrieval_config extras (region, embedding model, + # bucket, credentials refs, ...) to the search call; kwargs win on conflict. + provider_search_params = {k: v for k, v in retrieval_config.items() if k not in _CONSUMED_RETRIEVAL_CONFIG_KEYS} with _suppressed_sub_call_billing(): search_response = await litellm.vector_stores.asearch( vector_store_id=retrieval_config["vector_store_id"], query=query_text, max_num_results=retrieval_config.get("top_k", 10), custom_llm_provider=retrieval_config.get("custom_llm_provider", "openai"), - **kwargs, + router=router, + **{**provider_search_params, **kwargs}, ) search_provider = retrieval_config.get("custom_llm_provider", "openai") diff --git a/litellm/router.py b/litellm/router.py index 78fe3ff025e..bffd1df3814 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5820,6 +5820,7 @@ class Router: return await self._init_vector_store_api_endpoints( original_function=original_function, custom_llm_provider=custom_llm_provider, + call_type=call_type, **kwargs, ) elif call_type in ("afile_delete", "afile_content"): @@ -5860,6 +5861,7 @@ class Router: self, original_function: Callable, custom_llm_provider: Optional[str] = None, + call_type: Optional[str] = None, **kwargs, ): """ @@ -5878,6 +5880,12 @@ class Router: **kwargs, ) + # For search, pass the router so provider transforms can resolve + # router-managed embedding models (e.g. S3 Vectors query embeddings). + # Assigning into kwargs also overrides any client-supplied `router` key. + if call_type == "avector_store_search": + kwargs["router"] = self + # Otherwise, call the original function directly return await original_function(**kwargs) diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index f768ee75545..4035125120e 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -6,7 +6,7 @@ import asyncio import builtins import contextvars from functools import partial -from typing import Any, Coroutine, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Union import httpx @@ -28,6 +28,9 @@ from litellm.types.vector_stores import ( from litellm.utils import ProviderConfigManager, client from litellm.vector_stores.utils import VectorStoreRequestUtils +if TYPE_CHECKING: + from litellm.router import Router + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -279,6 +282,7 @@ async def asearch( timeout: Optional[Union[float, httpx.Timeout]] = None, # LiteLLM specific params, custom_llm_provider: Optional[str] = None, + router: Optional["Router"] = None, **kwargs, ) -> VectorStoreSearchResponse: """ @@ -307,6 +311,7 @@ async def asearch( extra_body=extra_body, timeout=timeout, custom_llm_provider=custom_llm_provider, + router=router, **kwargs, ) @@ -346,6 +351,7 @@ def search( timeout: Optional[Union[float, httpx.Timeout]] = None, # LiteLLM specific params, custom_llm_provider: Optional[str] = None, + router: Optional["Router"] = None, **kwargs, ) -> Union[VectorStoreSearchResponse, Coroutine[Any, Any, VectorStoreSearchResponse]]: """ @@ -449,6 +455,7 @@ def search( timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), + router=router, ) return response diff --git a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py index 7085e45cdc3..9389476bef4 100644 --- a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py +++ b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, Mock +from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest @@ -9,6 +9,18 @@ from litellm.llms.s3_vectors.vector_stores.transformation import ( from litellm.types.vector_stores import VectorStoreSearchResponse +def _mock_router(model_names, sync=False): + """Router mock serving the given embedding model names.""" + router = MagicMock() + router.get_model_list.return_value = [{"model_name": name} for name in model_names] + embedding_response = Mock(data=[{"embedding": [0.1, 0.2, 0.3]}]) + if sync: + router.embedding = MagicMock(return_value=embedding_response) + else: + router.aembedding = AsyncMock(return_value=embedding_response) + return router + + class TestS3VectorsVectorStoreConfig: def test_init(self): """Test that S3VectorsVectorStoreConfig initializes correctly""" @@ -28,19 +40,174 @@ class TestS3VectorsVectorStoreConfig: url = config.get_complete_url(None, litellm_params) assert url == "https://s3vectors.us-west-2.api.aws" - def test_get_complete_url_missing_region(self): - """Test that missing region raises error""" + def test_get_complete_url_missing_region(self, monkeypatch): + """Missing region falls back to the default region (parity with ingestion)""" + monkeypatch.delenv("AWS_REGION_NAME", raising=False) + monkeypatch.delenv("AWS_REGION", raising=False) config = S3VectorsVectorStoreConfig() - litellm_params = {} - with pytest.raises(ValueError, match="aws_region_name is required"): - config.get_complete_url(None, litellm_params) + url = config.get_complete_url(None, {}) + assert url == "https://s3vectors.us-west-2.api.aws" + + def test_get_complete_url_uses_env_region(self, monkeypatch): + """Missing region param resolves from AWS_REGION_NAME env var""" + monkeypatch.setenv("AWS_REGION_NAME", "eu-west-1") + monkeypatch.delenv("AWS_REGION", raising=False) + config = S3VectorsVectorStoreConfig() + url = config.get_complete_url(None, {}) + assert url == "https://s3vectors.eu-west-1.api.aws" + + def test_get_complete_url_invalid_region_format(self): + """Invalid region format raises""" + config = S3VectorsVectorStoreConfig() + with pytest.raises(ValueError, match="Invalid AWS region format"): + config.get_complete_url(None, {"aws_region_name": "Bad_Region!"}) - @pytest.mark.skip(reason="Requires embedding API call, tested in integration tests") def test_transform_search_request(self): - """Test search request transformation""" - # This test requires making an actual embedding API call - # It's better tested in integration tests - pass + """Full request-body transformation with a router-injected embedding""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["text-embedding-3-small"], sync=True) + + url, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={"max_num_results": 7}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + router=router, + ) + + assert url == "https://s3vectors.us-west-2.api.aws/QueryVectors" + assert request_body == { + "vectorBucketName": "test-bucket", + "indexName": "test-index", + "queryVector": {"float32": [0.1, 0.2, 0.3]}, + "topK": 7, + "returnDistance": True, + "returnMetadata": True, + } + assert mock_logging_obj.model_call_details["query"] == "test query" + + @pytest.mark.asyncio + async def test_atransform_search_uses_router_for_virtual_model(self): + """Regression: router-served embedding models must resolve via the router, + not a bare litellm.aembedding call (which has no deployment credentials).""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["my-embedding-model"]) + + with patch("litellm.aembedding", new=AsyncMock()) as mock_bare_aembedding: + url, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "my-embedding-model"}, + extra_body=None, + router=router, + ) + + router.aembedding.assert_awaited_once_with(model="my-embedding-model", input=["test query"]) + mock_bare_aembedding.assert_not_awaited() + assert request_body["queryVector"]["float32"] == [0.1, 0.2, 0.3] + assert request_body["topK"] == 5 # default + + @pytest.mark.asyncio + async def test_atransform_search_falls_back_when_router_does_not_serve_model(self): + """Router present but embedding_model is not a router deployment -> + bare litellm.aembedding keeps working (provider-prefixed + env creds stores).""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["some-other-model"]) + + mock_bare = AsyncMock(return_value=Mock(data=[{"embedding": [0.4, 0.5]}])) + with patch("litellm.aembedding", new=mock_bare): + _, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "azure/text-embedding-3-small"}, + extra_body=None, + router=router, + ) + + mock_bare.assert_awaited_once_with(model="azure/text-embedding-3-small", input=["test query"]) + router.aembedding.assert_not_awaited() + assert request_body["queryVector"]["float32"] == [0.4, 0.5] + + @pytest.mark.asyncio + async def test_atransform_search_without_router_uses_bare_embedding(self): + """Backward compat: no router -> bare litellm.aembedding as before""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + + mock_bare = AsyncMock(return_value=Mock(data=[{"embedding": [0.6, 0.7]}])) + with patch("litellm.aembedding", new=mock_bare): + _, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + ) + + mock_bare.assert_awaited_once_with(model="text-embedding-3-small", input=["test query"]) + assert request_body["queryVector"]["float32"] == [0.6, 0.7] + + def test_transform_search_uses_router_for_virtual_model_sync(self): + """Sync twin: router-served embedding model resolves via router.embedding""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["my-embedding-model"], sync=True) + + with patch("litellm.embedding", new=MagicMock()) as mock_bare_embedding: + _, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "my-embedding-model"}, + extra_body=None, + router=router, + ) + + router.embedding.assert_called_once_with(model="my-embedding-model", input=["test query"]) + mock_bare_embedding.assert_not_called() + assert request_body["queryVector"]["float32"] == [0.1, 0.2, 0.3] + + def test_transform_search_without_router_uses_bare_embedding_sync(self): + """Sync twin: no router -> bare litellm.embedding as before""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + + mock_bare = MagicMock(return_value=Mock(data=[{"embedding": [0.8, 0.9]}])) + with patch("litellm.embedding", new=mock_bare): + _, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + ) + + mock_bare.assert_called_once_with(model="text-embedding-3-small", input=["test query"]) + assert request_body["queryVector"]["float32"] == [0.8, 0.9] def test_transform_search_request_invalid_vector_store_id(self): """Test that invalid vector_store_id format raises error""" diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 15a117bd6fc..8bd67754952 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -327,3 +327,106 @@ def test_rag_query_stream_returns_event_stream(client_internal_user): assert response.headers.get("content-type", "").startswith("text/event-stream") assert '"object":"chat.completion.chunk"' in response.text assert "data: [DONE]" in response.text + + +def test_rag_query_merges_managed_store_params(client_internal_user): + """ + Regression: /v1/rag/query must consult the managed vector store registry + (like the direct /v1/vector_stores/{id}/search endpoint does) so that + provider, region, embedding model, etc. don't have to be repeated in + retrieval_config. Pre-fix the registry was never read, so managed S3 + Vectors stores failed with "aws_region_name is required". + """ + import litellm + from litellm.types.utils import ModelResponse + + mock_vector_store = { + "vector_store_id": "s3-store", + "custom_llm_provider": "s3_vectors", + "litellm_params": { + "aws_region_name": "eu-west-1", + "embedding_model": "my-embed", + "vector_bucket_name": "bkt", + }, + } + mock_registry = MagicMock() + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = mock_vector_store + + mock_response = ModelResponse( + id="chatcmpl-test", + choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + model="gpt-4o-mini", + ) + + with patch( + "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( + "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", + new=AsyncMock(), + ), patch( + "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", + new=AsyncMock(), + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store"}, + }, + ) + + assert response.status_code == 200, response.json() + mock_aquery.assert_awaited_once() + forwarded_config = mock_aquery.await_args.kwargs["retrieval_config"] + assert forwarded_config["vector_store_id"] == "s3-store" + assert forwarded_config["custom_llm_provider"] == "s3_vectors" + assert forwarded_config["aws_region_name"] == "eu-west-1" + assert forwarded_config["embedding_model"] == "my-embed" + assert forwarded_config["vector_bucket_name"] == "bkt" + + +def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): + """User-supplied retrieval_config keys must win over registry values.""" + import litellm + from litellm.types.utils import ModelResponse + + mock_vector_store = { + "vector_store_id": "s3-store", + "custom_llm_provider": "s3_vectors", + "litellm_params": {"aws_region_name": "eu-west-1"}, + } + mock_registry = MagicMock() + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = mock_vector_store + + mock_response = ModelResponse( + id="chatcmpl-test", + choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + model="gpt-4o-mini", + ) + + with patch( + "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( + "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", + new=AsyncMock(), + ), patch( + "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", + new=AsyncMock(), + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store", "aws_region_name": "us-east-1"}, + }, + ) + + assert response.status_code == 200, response.json() + forwarded_config = mock_aquery.await_args.kwargs["retrieval_config"] + assert forwarded_config["aws_region_name"] == "us-east-1" diff --git a/tests/test_litellm/rag/test_main.py b/tests/test_litellm/rag/test_main.py index 584124ba06a..d8ffae667b1 100644 --- a/tests/test_litellm/rag/test_main.py +++ b/tests/test_litellm/rag/test_main.py @@ -254,6 +254,96 @@ async def test_aquery_streaming_bills_sub_call_costs_into_final_event(): assert standard_logging_object["response_cost"] >= 0.003 +@pytest.mark.asyncio +async def test_aquery_forwards_provider_retrieval_config_and_router_to_search(): + """ + Regression: provider-specific retrieval_config keys (aws_region_name, + embedding_model, vector_bucket_name, ...) and the router must be forwarded + to the vector store search call. Pre-fix they were silently dropped, so + /v1/rag/query failed with provider config errors (e.g. S3 Vectors + "aws_region_name is required") even when the caller supplied them. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}, + } + ] + ) + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): + response = await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={ + "vector_store_id": "bkt:idx", + "custom_llm_provider": "s3_vectors", + "top_k": 5, + "aws_region_name": "eu-west-1", + "embedding_model": "my-embed", + "vector_bucket_name": "bkt", + }, + router=router, + mock_response="hi", + ) + + assert isinstance(response, ModelResponse) + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["vector_store_id"] == "bkt:idx" + assert search_kwargs["custom_llm_provider"] == "s3_vectors" + assert search_kwargs["max_num_results"] == 5 + assert search_kwargs["router"] is router + # provider-specific extras forwarded + assert search_kwargs["aws_region_name"] == "eu-west-1" + assert search_kwargs["embedding_model"] == "my-embed" + assert search_kwargs["vector_bucket_name"] == "bkt" + # consumed keys are not duplicated into the spread + assert "top_k" not in search_kwargs + + +@pytest.mark.asyncio +async def test_aquery_minimal_retrieval_config_forwards_no_extras(): + """ + A minimal retrieval_config must not leak consumed keys (or invent extras) + into the vector store search call. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): + await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={"vector_store_id": "vs_test_123", "custom_llm_provider": "openai"}, + mock_response="hi", + ) + + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["vector_store_id"] == "vs_test_123" + assert search_kwargs["custom_llm_provider"] == "openai" + assert search_kwargs["router"] is None + leaked = {"top_k", "filters", "retrieval_filter", "aws_region_name", "embedding_model", "vector_bucket_name"} + assert not (leaked & set(search_kwargs.keys())) + + def test_rag_call_types_are_registered(): """ query/aquery/ingest/aingest are @client-decorated entry points, so their diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index a9e5b3316e0..91d2973af76 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5936,3 +5936,58 @@ async def test_acreate_batch_request_bedrock_tags_override_deployment_tags(): bedrock_tags=request_tags, ) assert mock_sign.call_args.kwargs["data"]["tags"] == request_tags + + +@pytest.mark.asyncio +async def test_avector_store_search_injects_router(): + """ + Regression: router.avector_store_search must pass the router down to the + SDK search call so provider transforms can resolve router-managed + embedding models (e.g. S3 Vectors query embeddings). + """ + from litellm.types.vector_stores import VectorStoreSearchResponse + + mock_asearch = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + # Router.__init__ binds asearch via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.asearch", new=mock_asearch): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + await router.avector_store_search( + vector_store_id="v", query="q", custom_llm_provider="s3_vectors" + ) + + mock_asearch.assert_awaited_once() + assert mock_asearch.await_args.kwargs["router"] is router + + +@pytest.mark.asyncio +async def test_avector_store_create_does_not_inject_router(): + """The router injection is gated on the search call type: the create path + must keep calling the SDK without a router kwarg.""" + mock_acreate = AsyncMock(return_value={"id": "vs_1", "object": "vector_store"}) + # avector_store_create(model=None) resolves acreate via a local import at + # call time, so patching after Router construction works here. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + with patch("litellm.vector_stores.main.acreate", new=mock_acreate): + await router.avector_store_create(model=None, custom_llm_provider="openai") + + mock_acreate.assert_awaited_once() + assert "router" not in mock_acreate.await_args.kwargs diff --git a/tests/test_litellm/vector_stores/test_main.py b/tests/test_litellm/vector_stores/test_main.py new file mode 100644 index 00000000000..3fdf4d9daa5 --- /dev/null +++ b/tests/test_litellm/vector_stores/test_main.py @@ -0,0 +1,77 @@ +""" +Tests for litellm/vector_stores/main.py. + +Pins the router threading contract for vector store search: the router is an +explicit named parameter that reaches the HTTP handler, and it must never leak +into litellm_params/kwargs where logging would model_dump() it (the #19550 +serialization trap). +""" + +from unittest.mock import MagicMock, patch + +import litellm.vector_stores.main as vector_stores_main +from litellm.vector_stores.main import search + +MOCK_SEARCH_RESPONSE = { + "object": "vector_store.search_results.page", + "search_query": "q", + "data": [], +} + + +def test_search_threads_router_to_handler(): + """search() must pass its router param through to the HTTP handler""" + mock_router = MagicMock() + logger = MagicMock() + + with ( + patch( + "litellm.vector_stores.main.ProviderConfigManager.get_provider_vector_stores_config", + return_value=MagicMock(), + ), + patch.object( + vector_stores_main.base_llm_http_handler, + "vector_store_search_handler", + return_value=MOCK_SEARCH_RESPONSE, + ) as mock_handler, + ): + search( + vector_store_id="bkt:idx", + query="q", + custom_llm_provider="s3_vectors", + router=mock_router, + litellm_logging_obj=logger, + ) + + mock_handler.assert_called_once() + assert mock_handler.call_args.kwargs["router"] is mock_router + + +def test_search_router_not_in_litellm_params(): + """Regression (#19550 class): the router must stay out of GenericLiteLLMParams, + otherwise pre-call logging model_dump()s it and breaks serialization.""" + mock_router = MagicMock() + logger = MagicMock() + + with ( + patch( + "litellm.vector_stores.main.ProviderConfigManager.get_provider_vector_stores_config", + return_value=MagicMock(), + ), + patch.object( + vector_stores_main.base_llm_http_handler, + "vector_store_search_handler", + return_value=MOCK_SEARCH_RESPONSE, + ) as mock_handler, + ): + search( + vector_store_id="bkt:idx", + query="q", + custom_llm_provider="s3_vectors", + router=mock_router, + litellm_logging_obj=logger, + ) + + litellm_params = mock_handler.call_args.kwargs["litellm_params"] + assert "router" not in litellm_params.model_dump(exclude_none=True) + assert getattr(litellm_params, "router", None) is None diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx index cbabcc6dca5..f375bfdd351 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx @@ -128,16 +128,33 @@ describe("VectorStoreTester", () => { await waitFor(() => expect(mockSearch).toHaveBeenCalledTimes(1)); }); - it("reports a failed search and keeps the history empty", async () => { + it("shows the backend error in the history when a search fails", async () => { const user = userEvent.setup(); - mockSearch.mockRejectedValue(new Error("boom")); + const errorBody = '{"error":{"message":"OpenAIException - api_key is required"}}'; + mockSearch.mockRejectedValue(new Error(errorBody)); renderTester(); await user.type(queryInput(), "hello"); await user.click(searchButton()); - await waitFor(() => expect(mockFromBackend).toHaveBeenCalledWith("Failed to search vector store")); - expect(screen.getByText(EMPTY_STATE)).toBeInTheDocument(); + await waitFor(() => expect(mockFromBackend).toHaveBeenCalledWith(errorBody)); + expect(screen.getByText(`Search failed: ${errorBody}`)).toBeInTheDocument(); + expect(screen.queryByText("No results found")).not.toBeInTheDocument(); + expect(screen.queryByText(EMPTY_STATE)).not.toBeInTheDocument(); + // the failed query stays in the input for retry + expect(queryInput()).toHaveValue("hello"); + }); + + it('renders "No results found" for an empty result set, not an error', async () => { + const user = userEvent.setup(); + mockSearch.mockResolvedValue({ object: "vector_store.search_results.page", search_query: "hello", data: [] }); + renderTester(); + + await user.type(queryInput(), "hello"); + await user.click(searchButton()); + + expect(await screen.findByText("No results found")).toBeInTheDocument(); + expect(screen.queryByText(/search failed/i)).not.toBeInTheDocument(); }); it("clears the search history", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx index 015d58e8649..65b31bb911a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx @@ -41,6 +41,7 @@ export const VectorStoreTester: React.FC = ({ vectorStor { query: string; response: VectorStoreSearchResponse | null; + error: string | null; timestamp: number; }[] >([]); @@ -60,6 +61,7 @@ export const VectorStoreTester: React.FC = ({ vectorStor const historyEntry = { query, response, + error: null, timestamp: Date.now(), }; @@ -67,7 +69,9 @@ export const VectorStoreTester: React.FC = ({ vectorStor setQuery(""); } catch (error) { console.error("Error searching vector store:", error); - NotificationsManager.fromBackend("Failed to search vector store"); + const errorMessage = error instanceof Error ? error.message : String(error); + NotificationsManager.fromBackend(errorMessage); + setSearchHistory((prev) => [{ query, response: null, error: errorMessage, timestamp: Date.now() }, ...prev]); } finally { setIsLoading(false); } @@ -228,6 +232,8 @@ export const VectorStoreTester: React.FC = ({ vectorStor ); })} + ) : entry.error ? ( +
Search failed: {entry.error}
) : (
No results found
)} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 576e16cbb37..a8408fecaad 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -6851,7 +6851,7 @@ export const vectorStoreSearchCall = async ( if (!response.ok) { const errorData = await response.text(); await handleError(errorData); - return null; + throw new Error(errorData); } const data = await response.json(); From 779441b47ba564696c35e86a66527e683c1fad31 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:30:19 +0000 Subject: [PATCH 002/175] fix(bedrock_mantle): source per-request AWS credential params from litellm_params when signing chat completions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/custom_httpx/llm_http_handler.py | 26 ++++++- .../test_bedrock_mantle_transformation.py | 77 +++++++++++++++++++ .../custom_httpx/test_llm_http_handler.py | 19 +++++ 3 files changed, 120 insertions(+), 2 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 721b9545ac1..dfc234d08e0 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5,7 +5,7 @@ import ssl from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping from contextlib import asynccontextmanager from functools import lru_cache -from types import ModuleType +from types import MappingProxyType, ModuleType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, TypeVar, Union, cast, get_type_hints from urllib.parse import parse_qs, urlencode, urlparse, urlunparse @@ -20,6 +20,7 @@ from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.anthropic_messages.transformation import ( @@ -252,6 +253,24 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool: return False +def _aws_signing_overrides( + optional_params: Mapping[str, Any], litellm_params: Mapping[str, Any] +) -> Mapping[str, Any]: + """AWS credential params for SigV4 signers that read them off optional_params. + + Only `bedrock`/`sagemaker` keep `aws_*` in optional_params: every other provider + spreads optional_params into the request body, so the params are stripped there + and survive on litellm_params alone. + """ + return MappingProxyType( + { + key: litellm_params[key] + for key in AWS_CREDENTIAL_KWARGS_KEYS + if optional_params.get(key) is None and litellm_params.get(key) is not None + } + ) + + class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -495,7 +514,10 @@ class BaseLLMHTTPHandler: headers, signed_json_body = provider_config.sign_request( headers=headers, - optional_params=optional_params, + optional_params={ + **optional_params, + **_aws_signing_overrides(optional_params, litellm_params), + }, request_data=data, api_base=api_base, api_key=api_key, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 275fb460b9f..468cc9b9130 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -489,6 +489,83 @@ class TestBedrockMantleChatAuth: assert "/us-east-2/bedrock/aws4_request" in authorization assert requests[0]["url"].startswith("https://bedrock-mantle.us-east-2.api.aws") + def test_completion_per_request_role_reaches_signer_and_not_the_body( + self, monkeypatch + ): + # Per-request aws_role_name/aws_session_name are stripped from optional_params + # for non-bedrock providers, so they must be sourced from litellm_params at + # signing time, and must never be serialized into the provider request body. + from botocore.credentials import Credentials + + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + for var in ( + "BEDROCK_MANTLE_API_KEY", + "AWS_BEARER_TOKEN_BEDROCK", + "BEDROCK_MANTLE_API_BASE", + ): + monkeypatch.delenv(var, raising=False) + + credential_calls = [] + + def fake_get_credentials(self, **kwargs): + credential_calls.append(kwargs) + return Credentials( + access_key="ASIAEXAMPLE", + secret_key="YXNzdW1lZC1yb2xlLXNlY3JldC1hc3N1bWVk", + token="assumed-session-token", + ) + + monkeypatch.setattr(BaseAWSLLM, "get_credentials", fake_get_credentials) + + requests = [] + + def mock_post(self, url, data=None, headers=None, **kwargs): + raw_body = data.decode("utf-8") if isinstance(data, bytes) else data + requests.append({"headers": headers or {}, "body": json.loads(raw_body or "{}")}) + return httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1733529600, + "model": "google.gemma-4-31b", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + }, + request=httpx.Request("POST", url), + ) + + with patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post + ): + litellm.completion( + model="bedrock_mantle/google.gemma-4-31b", + messages=[{"role": "user", "content": "hello"}], + aws_role_name="arn:aws:iam::000000000000:role/attributed-role", + aws_session_name="user-123", + aws_region_name="us-east-1", + ) + + assert len(credential_calls) == 1 + assert ( + credential_calls[0]["aws_role_name"] + == "arn:aws:iam::000000000000:role/attributed-role" + ) + assert credential_calls[0]["aws_session_name"] == "user-123" + assert requests[0]["headers"]["Authorization"].startswith("AWS4-HMAC-SHA256") + assert not [key for key in requests[0]["body"] if key.startswith("aws_")] + class TestBedrockMantleProjectHeader: def test_validate_environment_sets_openai_project_header(self): diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index fddd8d09dfc..0906b39c514 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -2071,3 +2071,22 @@ async def test_anthropic_invalid_thinking_signature_retry_resigns_bedrock_reques retry_authorization = posts[1]["headers"]["Authorization"] assert retry_authorization.startswith("AWS4-HMAC-SHA256") assert retry_authorization != first_attempt_headers["Authorization"] + + +def test_aws_signing_overrides_only_fills_missing_credentials(): + from litellm.llms.custom_httpx.llm_http_handler import _aws_signing_overrides + + overrides = _aws_signing_overrides( + {"temperature": 0.2, "aws_region_name": "us-west-2"}, + { + "aws_role_name": "arn:aws:iam::000000000000:role/attributed", + "aws_session_name": "user-123", + "aws_region_name": "us-east-1", + "api_key": "not-an-aws-param", + }, + ) + + assert dict(overrides) == { + "aws_role_name": "arn:aws:iam::000000000000:role/attributed", + "aws_session_name": "user-123", + } From 2c7de60692d7a4fcd53964872d4355042d52b90b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:53:40 +0000 Subject: [PATCH 003/175] style: ruff format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/custom_httpx/llm_http_handler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index dfc234d08e0..cadf4c701e2 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -253,9 +253,7 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool: return False -def _aws_signing_overrides( - optional_params: Mapping[str, Any], litellm_params: Mapping[str, Any] -) -> Mapping[str, Any]: +def _aws_signing_overrides(optional_params: Mapping[str, Any], litellm_params: Mapping[str, Any]) -> Mapping[str, Any]: """AWS credential params for SigV4 signers that read them off optional_params. Only `bedrock`/`sagemaker` keep `aws_*` in optional_params: every other provider From 00a20591746b1d40f9c0ec5407ea06efb5a31234 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 13 Aug 2026 19:51:34 +0000 Subject: [PATCH 004/175] fix(router): resolve realtime session model to routed deployment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 22 +++++++++ tests/test_litellm/test_router.py | 76 +++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index fb2af41dcf2..95509bcdbba 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -29,6 +29,7 @@ import anyio import httpx import openai from openai import AsyncOpenAI +from pydantic import TypeAdapter, ValidationError from typing_extensions import overload import litellm @@ -342,6 +343,26 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) return False +_NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({}) +_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +def _with_router_resolved_session_model(session: object, model_name: str) -> Mapping[str, Mapping[str, object]]: + """ + Realtime client-secret requests carry the model inside ``session`` as well, and the caller's copy of it still + holds the pre-routing model group name, so it has to follow the deployment the router just picked. + + Returns kwargs to merge into the downstream call, empty when there is no session model to resolve. + """ + try: + typed_session: Final = _SESSION_ADAPTER.validate_python(session) + except ValidationError: + return _NO_SESSION_KWARGS + if "model" not in typed_session: + return _NO_SESSION_KWARGS + return MappingProxyType({"session": {**typed_session, "model": model_name}}) + + class RoutingArgs(enum.Enum): ttl = 60 # 1min (RPM/TPM expire key) @@ -4685,6 +4706,7 @@ class Router: "caching": self.cache_responses, **kwargs, "model": model_name, + **_with_router_resolved_session_model(kwargs.get("session"), model_name), } # Only set custom_llm_provider if it's not None if custom_llm_provider is not None: diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index bdbf33fb0e1..fbefcc59477 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1298,6 +1298,82 @@ async def test_ageneric_api_call_deployment_model_overrides_alias(): ), f"Expected deployment model 'vertex_ai/gemini-2.5-flash', got '{captured['model']}'" +@pytest.mark.asyncio +async def test_ageneric_api_call_resolves_realtime_session_model(): + """ + Regression for #36742: realtime client secret requests carry the model inside `session` too, and the proxy + fills it with the pre-routing model group name. The underlying litellm function reads session.model first, + so it must see the resolved deployment, while a caller's nested transcription model stays untouched. + """ + captured: dict = {} + + async def capture_kwargs(**kwargs): + captured.update(kwargs) + return {"result": "ok"} + + router = litellm.Router( + model_list=[ + { + "model_name": "my-realtime-group", + "litellm_params": { + "model": "openai/gpt-realtime-2.1-mini", + "api_key": "fake-key", + }, + "model_info": {"mode": "realtime"}, + } + ] + ) + + await router._ageneric_api_call_with_fallbacks( + model="my-realtime-group", + original_function=capture_kwargs, + session={ + "type": "realtime", + "model": "my-realtime-group", + "audio": {"input": {"transcription": {"model": "gpt-4o-transcribe"}}}, + }, + ) + + assert captured["model"] == "openai/gpt-realtime-2.1-mini" + assert captured["session"]["model"] == "openai/gpt-realtime-2.1-mini" + assert captured["session"]["audio"]["input"]["transcription"]["model"] == "gpt-4o-transcribe" + + +@pytest.mark.asyncio +async def test_ageneric_api_call_does_not_add_session_model(): + """ + A session that never carried a model must not gain one from routing: the underlying function then falls back + to the resolved `model` kwarg itself, and the outgoing session body keeps the caller's shape. + """ + captured: dict = {} + + async def capture_kwargs(**kwargs): + captured.update(kwargs) + return {"result": "ok"} + + router = litellm.Router( + model_list=[ + { + "model_name": "my-realtime-group", + "litellm_params": { + "model": "openai/gpt-realtime-2.1-mini", + "api_key": "fake-key", + }, + "model_info": {"mode": "realtime"}, + } + ] + ) + + await router._ageneric_api_call_with_fallbacks( + model="my-realtime-group", + original_function=capture_kwargs, + session={"type": "realtime"}, + ) + + assert captured["model"] == "openai/gpt-realtime-2.1-mini" + assert captured["session"] == {"type": "realtime"} + + def test_router_get_model_access_groups_team_only_models(): """ Test that Router.get_model_access_groups returns the correct response for team-only models From e5b54620a4fcf67d13fb5aa2261dab31502be5ab Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 13 Aug 2026 19:58:03 +0000 Subject: [PATCH 005/175] test(router): cover realtime session model resolver directly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_router.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index fbefcc59477..99e7e9ad6bc 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1374,6 +1374,21 @@ async def test_ageneric_api_call_does_not_add_session_model(): assert captured["session"] == {"type": "realtime"} +@pytest.mark.parametrize( + "session, expected", + [ + ({"type": "realtime", "model": "my-realtime-group"}, {"session": {"type": "realtime", "model": "resolved"}}), + ({"type": "realtime"}, {}), + (None, {}), + ("not-a-session", {}), + ], +) +def test_with_router_resolved_session_model(session, expected): + from litellm.router import _with_router_resolved_session_model + + assert dict(_with_router_resolved_session_model(session, "resolved")) == expected + + def test_router_get_model_access_groups_team_only_models(): """ Test that Router.get_model_access_groups returns the correct response for team-only models From 3275459aec0935b03b60950ba5d625854e2d6c9b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:47:26 -0700 Subject: [PATCH 006/175] fix(mcp): cap tools preview and test-connection at the listing timeout and name the unreachable upstream --- .../mcp_server/rest_endpoints.py | 34 +++++--- .../mcp_server/test_rest_endpoints.py | 77 +++++++++++++++++-- 2 files changed, 96 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 3efb6429326..d73277f4417 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -4,10 +4,12 @@ from collections.abc import Awaitable, Callable, Mapping from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal +import anyio import httpx from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from litellm._logging import verbose_logger +from litellm.constants import MCP_TOOL_LISTING_TIMEOUT from litellm.exceptions import ( BlockedPiiEntityError, GuardrailRaisedException, @@ -68,7 +70,13 @@ _MCP_GUARDRAIL_REJECTIONS: Final = ( ) -def _connection_error_message(exc: BaseException) -> str: +def _connection_error_message(exc: BaseException, url: str | None, timeout_seconds: float) -> str: + if isinstance(exc, TimeoutError): + return ( + f"Failed to connect to MCP server: no response from {url or 'the server'} " + f"within {timeout_seconds:.0f}s. Check that the LiteLLM proxy can reach this URL " + "from its network (DNS, egress rules, firewalls) and that the server answers MCP requests." + ) if isinstance(exc, httpx.LocalProtocolError): return ( "Failed to connect to MCP server: a request header is malformed. " @@ -1136,6 +1144,7 @@ if MCP_AVAILABLE: mcp_auth_header: str | dict[str, str] | None = None, oauth2_headers: dict[str, str] | None = None, raw_headers: dict[str, str] | None = None, + timeout_seconds: float = MCP_TOOL_LISTING_TIMEOUT, ) -> Mapping[str, object]: """ Create a temporary MCP client from *request*, run *operation*, and return the result. @@ -1151,6 +1160,10 @@ if MCP_AVAILABLE: oauth2_headers: Headers extracted from the incoming request (may contain the litellm API key — must NOT be forwarded for M2M servers). raw_headers: Raw request headers forwarded for stdio env construction. + timeout_seconds: Cap on OAuth discovery, connect, handshake, and *operation* + combined. Defaults to ``MCP_TOOL_LISTING_TIMEOUT`` (30s, below common LB + timeouts) so an unreachable upstream yields this endpoint's JSON error + instead of an opaque load-balancer 504 with an empty body. Returns: The dict returned by *operation*, or an error dict on failure. @@ -1240,15 +1253,16 @@ if MCP_AVAILABLE: static_headers=request.static_headers, ) - client: Final = await global_mcp_server_manager._create_mcp_client( - server=server_model, - mcp_auth_header=mcp_auth_header, - extra_headers=merged_headers, - stdio_env=stdio_env, - cred_provider=preview_cred_provider, - ) + with anyio.fail_after(timeout_seconds): + client: Final = await global_mcp_server_manager._create_mcp_client( + server=server_model, + mcp_auth_header=mcp_auth_header, + extra_headers=merged_headers, + stdio_env=stdio_env, + cred_provider=preview_cred_provider, + ) - return await operation(client) + return await operation(client) except (KeyboardInterrupt, SystemExit, asyncio.CancelledError): raise @@ -1257,7 +1271,7 @@ if MCP_AVAILABLE: return { "status": "error", "error": True, - "message": _connection_error_message(e), + "message": _connection_error_message(e, request.url, timeout_seconds), } async def _preview_openapi_tools(spec_path: str) -> dict: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index ef5631218f3..51b946c11b7 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -1,4 +1,5 @@ import asyncio +import inspect import json import sys from datetime import datetime @@ -13,6 +14,7 @@ import pytest from fastapi import HTTPException from starlette.requests import Request +from litellm.constants import MCP_TOOL_LISTING_TIMEOUT from litellm.proxy._experimental.mcp_server import rest_endpoints from litellm.proxy._experimental.mcp_server.auth import ( user_api_key_auth_mcp as auth_mcp, @@ -109,6 +111,71 @@ class TestExecuteWithMcpClient: assert result["status"] == "error" assert "stack_trace" not in result + @pytest.mark.asyncio + async def test_timeout_caps_hanging_operation_and_names_url(self, monkeypatch): + async def fake_create_client(*args, **kwargs): + return object() + + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "_create_mcp_client", + fake_create_client, + ) + + async def hanging_operation(client): + await asyncio.Event().wait() + + payload = NewMCPServerRequest( + server_name="example", + url="https://mcp.example.com/mcp/", + auth_type=MCPAuth.none, + ) + + result = await asyncio.wait_for( + rest_endpoints._execute_with_mcp_client(payload, hanging_operation, timeout_seconds=0.05), + timeout=5, + ) + + assert result["error"] is True + assert "https://mcp.example.com/mcp/" in result["message"] + + @pytest.mark.asyncio + async def test_timeout_covers_client_creation(self, monkeypatch): + async def hanging_create_client(*args, **kwargs): + await asyncio.Event().wait() + + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "_create_mcp_client", + hanging_create_client, + ) + + async def unreached_operation(client): + return {"status": "ok"} + + payload = NewMCPServerRequest( + server_name="example", + url="https://mcp.example.com/mcp/", + auth_type=MCPAuth.none, + ) + + result = await asyncio.wait_for( + rest_endpoints._execute_with_mcp_client(payload, unreached_operation, timeout_seconds=0.05), + timeout=5, + ) + + assert result["error"] is True + assert "https://mcp.example.com/mcp/" in result["message"] + + def test_timeout_defaults_to_tool_listing_timeout(self): + default = inspect.signature(rest_endpoints._execute_with_mcp_client).parameters["timeout_seconds"].default + assert default == MCP_TOOL_LISTING_TIMEOUT + + def test_connection_error_message_timeout_names_url_and_budget(self): + message = rest_endpoints._connection_error_message(TimeoutError(), "https://api.example.com/mcp/", 30.0) + assert "https://api.example.com/mcp/" in message + assert "30s" in message + @pytest.mark.asyncio async def test_forwards_static_headers(self, monkeypatch): """Ensure static_headers are forwarded to the MCP client during test calls. @@ -2881,17 +2948,17 @@ class TestConnectionErrorMessage: secret = "Bearer sk-super-secret-token" exc = httpx.LocalProtocolError(f"Illegal header value b' {secret}'") - message = rest_endpoints._connection_error_message(exc) + message = rest_endpoints._connection_error_message(exc, "https://example.com", 30.0) assert "header" in message.lower() assert secret not in message def test_connect_error_points_at_reachability(self): - message = rest_endpoints._connection_error_message(httpx.ConnectError("All connection attempts failed")) + message = rest_endpoints._connection_error_message(httpx.ConnectError("All connection attempts failed"), "https://example.com", 30.0) assert "unreachable" in message.lower() def test_timeout_error_message(self): - message = rest_endpoints._connection_error_message(httpx.ConnectTimeout("timed out")) + message = rest_endpoints._connection_error_message(httpx.ConnectTimeout("timed out"), "https://example.com", 30.0) assert "unreachable" in message.lower() def test_http_status_error_includes_status_code(self): @@ -2901,11 +2968,11 @@ class TestConnectionErrorMessage: request=httpx.Request("POST", "http://x/"), response=response, ) - message = rest_endpoints._connection_error_message(exc) + message = rest_endpoints._connection_error_message(exc, "https://example.com", 30.0) assert "503" in message def test_unknown_error_falls_back_to_generic(self): - message = rest_endpoints._connection_error_message(RuntimeError("weird")) + message = rest_endpoints._connection_error_message(RuntimeError("weird"), "https://example.com", 30.0) assert "weird" not in message assert "proxy logs" in message.lower() From f4b5449c6a65cf167658fd5bb32695abda9633a2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:48:04 -0700 Subject: [PATCH 007/175] fix(openai_like): strip cache_control ttl before forwarding /v1/messages to non-Anthropic providers --- litellm/llms/anthropic/common_utils.py | 33 +++++ litellm/llms/openai_like/README.md | 5 +- .../openai_like/messages/transformation.py | 38 +++++- ..._like_anthropic_messages_transformation.py | 121 ++++++++++++++++++ 4 files changed, 195 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 681a8397f66..695e3a313ef 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1302,6 +1302,39 @@ def flatten_unencrypted_web_search_results_in_anthropic_messages( # mutable-ok: return [_flatten_web_search_results_in_message(m) for m in messages] # mutable-ok: JSON wire format +def _normalized_cache_control(cache_control: dict) -> dict: # mutable-ok: as sibling sanitizers + cache_type: Final = cache_control.get("type") + return {"type": cache_type if isinstance(cache_type, str) else "ephemeral"} # mutable-ok: JSON wire format + + +def _normalize_cache_control_value(value: object) -> object: + if isinstance(value, dict): + return normalize_cache_control_in_anthropic_payload(value) + if isinstance(value, list): + return [_normalize_cache_control_value(item) for item in value] # mutable-ok: JSON wire format + return value + + +def normalize_cache_control_in_anthropic_payload(payload: dict) -> dict: # mutable-ok: as sibling sanitizers + """ + Return a copy of an Anthropic /v1/messages payload with every + ``cache_control`` entry reduced to ``{"type": }``, + recursing through message content blocks, system blocks, and tools. + + Anthropic itself accepts prompt-caching extensions such as ``ttl``, but + strict non-Anthropic implementations of the Messages API validate the field + literally and reject the whole request (``cache_control.ttl: 1h is not + supported``, ``cache_control.type is required``), which 400s clients like + Claude Code that always send cache hints. Non-dict ``cache_control`` values + are dropped entirely. The caller's payload is never mutated. + """ + return { # mutable-ok: JSON wire format, as sibling sanitizers + key: _normalized_cache_control(value) if key == "cache_control" else _normalize_cache_control_value(value) + for key, value in payload.items() + if key != "cache_control" or isinstance(value, dict) + } + + def process_anthropic_headers(headers: httpx.Headers | dict) -> dict: openai_headers: Final = {} if "anthropic-ratelimit-requests-limit" in headers: diff --git a/litellm/llms/openai_like/README.md b/litellm/llms/openai_like/README.md index e9aaafe48a1..e1409b81c35 100644 --- a/litellm/llms/openai_like/README.md +++ b/litellm/llms/openai_like/README.md @@ -54,7 +54,10 @@ That's it! The provider will be automatically loaded and available. "constraints": { "temperature_max": 1.0, "temperature_min": 0.0, - "temperature_min_with_n_gt_1": 0.3 + "temperature_min_with_n_gt_1": 0.3, + // /v1/messages providers only: keep Anthropic cache_control extensions + // such as ttl instead of stripping them down to {"type": ...} + "cache_control_ttl": true }, // Optional: Special handling flags diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 11dc236064d..29973fe2101 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -1,11 +1,13 @@ from typing import Any, Final import litellm +from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.openai_like.json_loader import SimpleProviderConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" @@ -19,7 +21,9 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): ``"/v1/messages"``. The inbound Anthropic payload (system, cache_control, thinking, tools, ...) is forwarded essentially unchanged to ``{api_base}/v1/messages``, so Anthropic-only features that the - Anthropic->OpenAI translation would otherwise drop are preserved. Response + Anthropic->OpenAI translation would otherwise drop are preserved. The one + exception is ``cache_control``, whose Anthropic-only extensions (``ttl``) + are stripped unless ``supports_cache_control_ttl`` says otherwise. Response parsing and streaming are inherited from the native Anthropic config. """ @@ -53,6 +57,35 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): def should_filter_anthropic_beta_headers(self) -> bool: return False + def supports_cache_control_ttl(self) -> bool: + return False + + def transform_anthropic_messages_request( + self, + model: str, + messages: list[dict], # mutable-ok: matches dict-typed base signature + anthropic_messages_optional_request_params: dict, # mutable-ok: matches dict-typed base signature + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: matches dict-typed base signature + ) -> dict: # mutable-ok: matches dict-typed base signature + """ + Anthropic ignores prompt-caching hints it cannot honor, but strict + non-Anthropic implementations of the Messages API 400 the whole request + on Anthropic-only ``cache_control`` extensions (``cache_control.ttl: 1h + is not supported``), so unless the provider declares ttl support the + hints are reduced to their portable ``{"type": ...}`` core. + """ + request: Final = super().transform_anthropic_messages_request( + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + if self.supports_cache_control_ttl(): + return request + return normalize_cache_control_in_anthropic_payload(request) + def get_complete_url( self, api_base: str | None, @@ -91,6 +124,9 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig): def should_strip_billing_metadata(self) -> bool: return True + def supports_cache_control_ttl(self) -> bool: + return bool(self._provider.constraints.get("cache_control_ttl")) + def _resolve_api_key(self, api_key: str | None) -> str | None: return api_key or get_secret_str(self._provider.api_key_env) or litellm.api_key diff --git a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py index 33e677b000e..2cdd969b00f 100644 --- a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py @@ -317,3 +317,124 @@ def test_json_provider_messages_config_probes_capabilities_under_provider_slug() ) assert JSONProviderAnthropicMessagesConfig(provider).custom_llm_provider == "exampleprovider" assert OpenAILikeAnthropicMessagesConfig().custom_llm_provider == "anthropic" + + +def _cache_control_request_params() -> tuple[list, dict]: + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "write a regex for a US phone number", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + } + ] + optional_params = { + "max_tokens": 256, + "system": [ + { + "type": "text", + "text": "You are Claude Code.", + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + } + ], + "tools": [ + { + "name": "lookup", + "input_schema": {"type": "object"}, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + } + return messages, optional_params + + +def test_request_strips_cache_control_ttl_everywhere(config): + """Regression: Claude Code always sends ``cache_control: {type: ephemeral, + ttl: 1h}``, and strict non-Anthropic /v1/messages validators 400 the whole + request on the ttl extension (``cache_control.ttl: 1h is not supported``).""" + messages, optional_params = _cache_control_request_params() + + payload = config.transform_anthropic_messages_request( + model="some-model", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert payload["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + assert payload["system"][0]["cache_control"] == {"type": "ephemeral"} + assert payload["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert messages[0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + +def test_request_defaults_missing_cache_control_type_and_drops_non_dict(config): + payload = config.transform_anthropic_messages_request( + model="some-model", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "a", "cache_control": {"ttl": "1h"}}, + {"type": "text", "text": "b", "cache_control": None}, + ], + } + ], + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + blocks = payload["messages"][0]["content"] + assert blocks[0]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in blocks[1] + + +def test_native_anthropic_config_keeps_cache_control_ttl(): + """Anthropic itself accepts ttl, so the normalization must stay scoped to + the OpenAI-like passthrough and never reach the native Anthropic path.""" + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) + + messages, optional_params = _cache_control_request_params() + payload = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-20250514", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert payload["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert payload["system"][0]["cache_control"] == {"type": "ephemeral", "ttl": "5m"} + + +def test_json_provider_constraint_opts_into_cache_control_ttl(): + from litellm.llms.openai_like.json_loader import SimpleProviderConfig + from litellm.llms.openai_like.messages.transformation import ( + JSONProviderAnthropicMessagesConfig, + ) + + base_data = {"base_url": "https://api.example.com/v1", "api_key_env": "EXAMPLE_API_KEY"} + strict = JSONProviderAnthropicMessagesConfig(SimpleProviderConfig(slug="strictprov", data=base_data)) + lenient = JSONProviderAnthropicMessagesConfig( + SimpleProviderConfig(slug="lenientprov", data={**base_data, "constraints": {"cache_control_ttl": True}}) + ) + + def transform(provider_config): + messages, optional_params = _cache_control_request_params() + return provider_config.transform_anthropic_messages_request( + model="some-model", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert transform(strict)["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + assert transform(lenient)["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} From d3268e4e184f8b7cde6cce4473e14b1acedfc751 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:26:04 -0700 Subject: [PATCH 008/175] test(mcp): wrap over-long connection error message calls --- .../proxy/_experimental/mcp_server/test_rest_endpoints.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 51b946c11b7..e66101f6177 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -2954,11 +2954,15 @@ class TestConnectionErrorMessage: assert secret not in message def test_connect_error_points_at_reachability(self): - message = rest_endpoints._connection_error_message(httpx.ConnectError("All connection attempts failed"), "https://example.com", 30.0) + message = rest_endpoints._connection_error_message( + httpx.ConnectError("All connection attempts failed"), "https://example.com", 30.0 + ) assert "unreachable" in message.lower() def test_timeout_error_message(self): - message = rest_endpoints._connection_error_message(httpx.ConnectTimeout("timed out"), "https://example.com", 30.0) + message = rest_endpoints._connection_error_message( + httpx.ConnectTimeout("timed out"), "https://example.com", 30.0 + ) assert "unreachable" in message.lower() def test_http_status_error_includes_status_code(self): From 0baf376efd691e139902ca7933b83666ea459a41 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:07:05 -0700 Subject: [PATCH 009/175] fix(openai_like): scope cache_control normalization to Messages API locations Rewrite the sanitizer without recursion (the code-quality gate rejects new recursive functions) and only touch cache_control where the Messages API defines it: the request, system blocks, tools, message content blocks, and tool_result content. Application data such as tool_use.input and tool input_schema is left untouched even when it contains a cache_control key --- litellm/llms/anthropic/common_utils.py | 78 +++++++++++++++---- ..._like_anthropic_messages_transformation.py | 62 +++++++++++++++ 2 files changed, 124 insertions(+), 16 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 695e3a313ef..1ef14362601 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1302,37 +1302,83 @@ def flatten_unencrypted_web_search_results_in_anthropic_messages( # mutable-ok: return [_flatten_web_search_results_in_message(m) for m in messages] # mutable-ok: JSON wire format -def _normalized_cache_control(cache_control: dict) -> dict: # mutable-ok: as sibling sanitizers +def _normalized_cache_control(cache_control: object) -> dict[str, str] | None: # mutable-ok: JSON wire format + if not isinstance(cache_control, Mapping): + return None cache_type: Final = cache_control.get("type") return {"type": cache_type if isinstance(cache_type, str) else "ephemeral"} # mutable-ok: JSON wire format -def _normalize_cache_control_value(value: object) -> object: - if isinstance(value, dict): - return normalize_cache_control_in_anthropic_payload(value) - if isinstance(value, list): - return [_normalize_cache_control_value(item) for item in value] # mutable-ok: JSON wire format - return value +def _with_portable_cache_control(block: Mapping[str, object]) -> dict[str, object]: # mutable-ok: JSON wire format + if "cache_control" not in block: + return dict(block) # mutable-ok: JSON wire format + normalized: Final = _normalized_cache_control(block["cache_control"]) + rest: Final = {key: value for key, value in block.items() if key != "cache_control"} # mutable-ok: JSON wire format + return rest if normalized is None else {**rest, "cache_control": normalized} # mutable-ok: JSON wire format -def normalize_cache_control_in_anthropic_payload(payload: dict) -> dict: # mutable-ok: as sibling sanitizers +def _with_portable_cache_control_in_blocks(blocks: object) -> object: + if isinstance(blocks, str) or not isinstance(blocks, Sequence): + return blocks + return [ # mutable-ok: JSON wire format + _with_portable_cache_control(block) if isinstance(block, Mapping) else block for block in blocks + ] + + +def _with_portable_cache_control_in_content_block(block: object) -> object: + if not isinstance(block, Mapping): + return block + portable: Final = _with_portable_cache_control(block) + if portable.get("type") != "tool_result" or "content" not in portable: + return portable + return { # mutable-ok: JSON wire format + **portable, + "content": _with_portable_cache_control_in_blocks(portable["content"]), + } + + +def _with_portable_cache_control_in_message(message: object) -> object: + if not isinstance(message, Mapping) or "content" not in message: + return message + content: Final = message["content"] + if isinstance(content, str) or not isinstance(content, Sequence): + return message + return { # mutable-ok: JSON wire format + **message, + "content": [_with_portable_cache_control_in_content_block(block) for block in content], + } + + +def normalize_cache_control_in_anthropic_payload( # mutable-ok: JSON wire format + payload: Mapping[str, object], +) -> dict[str, object]: """ Return a copy of an Anthropic /v1/messages payload with every - ``cache_control`` entry reduced to ``{"type": }``, - recursing through message content blocks, system blocks, and tools. + ``cache_control`` entry reduced to ``{"type": }`` + at the places the Messages API defines it: the request itself, system + blocks, tools, message content blocks, and ``tool_result`` content blocks. + Application data such as ``tool_use.input`` and tool ``input_schema`` is + never touched, even when it happens to contain a ``cache_control`` key. Anthropic itself accepts prompt-caching extensions such as ``ttl``, but strict non-Anthropic implementations of the Messages API validate the field literally and reject the whole request (``cache_control.ttl: 1h is not supported``, ``cache_control.type is required``), which 400s clients like - Claude Code that always send cache hints. Non-dict ``cache_control`` values - are dropped entirely. The caller's payload is never mutated. + Claude Code that send cache hints. Non-dict ``cache_control`` values are + dropped entirely. The caller's payload is never mutated. """ - return { # mutable-ok: JSON wire format, as sibling sanitizers - key: _normalized_cache_control(value) if key == "cache_control" else _normalize_cache_control_value(value) - for key, value in payload.items() - if key != "cache_control" or isinstance(value, dict) + portable: Final = _with_portable_cache_control(payload) + scoped: Final = { # mutable-ok: JSON wire format + key: ( + _with_portable_cache_control_in_blocks(value) + if key in ("system", "tools") + else [_with_portable_cache_control_in_message(message) for message in value] + if key == "messages" and isinstance(value, Sequence) and not isinstance(value, str) + else value + ) + for key, value in portable.items() } + return scoped def process_anthropic_headers(headers: httpx.Headers | dict) -> dict: diff --git a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py index 2cdd969b00f..d325492914e 100644 --- a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py @@ -438,3 +438,65 @@ def test_json_provider_constraint_opts_into_cache_control_ttl(): assert transform(strict)["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} assert transform(lenient)["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + +def test_request_strips_ttl_only_where_the_messages_api_defines_cache_control(config): + """Regression: the sanitizer must only touch ``cache_control`` where the + Messages API defines it (request, system, tools, content blocks, tool_result + content), never application data such as ``tool_use.input`` or a tool's + ``input_schema`` that happens to contain a ``cache_control`` key.""" + tool_input = {"cache_control": {"type": "ephemeral", "ttl": "1h"}, "query": "x"} + input_schema = { + "type": "object", + "properties": {"cache_control": {"type": "string", "ttl": "1h"}}, + } + messages = [ + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "lookup", "input": tool_input}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + "content": [ + {"type": "text", "text": "result", "cache_control": {"type": "ephemeral", "ttl": "1h"}} + ], + }, + {"type": "text", "text": "plain string content stays", "cache_control": {"ttl": "1h"}}, + ], + }, + {"role": "user", "content": "a plain string message"}, + ] + optional_params = { + "max_tokens": 64, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + "tools": [ + { + "name": "lookup", + "input_schema": input_schema, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + } + + payload = config.transform_anthropic_messages_request( + model="some-model", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert payload["cache_control"] == {"type": "ephemeral"} + assert payload["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert payload["tools"][0]["input_schema"] == input_schema + assert payload["messages"][0]["content"][0]["input"] == tool_input + tool_result = payload["messages"][1]["content"][0] + assert tool_result["cache_control"] == {"type": "ephemeral"} + assert tool_result["content"][0]["cache_control"] == {"type": "ephemeral"} + assert payload["messages"][1]["content"][1]["cache_control"] == {"type": "ephemeral"} + assert payload["messages"][2] == {"role": "user", "content": "a plain string message"} From c32eb41aad3b7b087c7d0023a71876d3dea6511d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:17:13 -0700 Subject: [PATCH 010/175] feat(openai_like): let a passthrough deployment keep cache_control ttl via model_info.cache_control_ttl The supported_endpoints passthrough had no way to keep ttl for an upstream that honors it, so the deployment now opts in with model_info.cache_control_ttl: true, injected into the config the same way the providers.json constraint is for JSON providers --- .../messages/handler.py | 8 +- .../openai_like/messages/transformation.py | 16 +-- ...erimental_pass_through_messages_handler.py | 97 ++++++++++--------- ..._like_anthropic_messages_transformation.py | 17 ++++ 4 files changed, 84 insertions(+), 54 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 69985bcdaa3..b82903d6f87 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -99,6 +99,10 @@ def _deployment_passes_through_anthropic_messages(model_info: object) -> bool: return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints +def _deployment_supports_cache_control_ttl(model_info: object) -> bool: + return isinstance(model_info, dict) and model_info.get("cache_control_ttl") is True + + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -568,7 +572,9 @@ def anthropic_messages_handler( OpenAILikeAnthropicMessagesConfig, ) - anthropic_messages_provider_config = OpenAILikeAnthropicMessagesConfig() + anthropic_messages_provider_config = OpenAILikeAnthropicMessagesConfig( + cache_control_ttl=_deployment_supports_cache_control_ttl(kwargs.get("model_info")), + ) if anthropic_messages_provider_config is None: # Route to Responses API for OpenAI / Azure, chat/completions for everything else. if _should_route_to_responses_api(custom_llm_provider, original_model, model): diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 29973fe2101..ac99617521c 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -23,10 +23,15 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): ``{api_base}/v1/messages``, so Anthropic-only features that the Anthropic->OpenAI translation would otherwise drop are preserved. The one exception is ``cache_control``, whose Anthropic-only extensions (``ttl``) - are stripped unless ``supports_cache_control_ttl`` says otherwise. Response - parsing and streaming are inherited from the native Anthropic config. + are stripped unless the deployment opts in with + ``model_info.cache_control_ttl: true``. Response parsing and streaming are + inherited from the native Anthropic config. """ + def __init__(self, cache_control_ttl: bool = False) -> None: + super().__init__() + self._cache_control_ttl: Final = cache_control_ttl + def validate_anthropic_messages_environment( self, headers: dict[str, str], @@ -58,7 +63,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): return False def supports_cache_control_ttl(self) -> bool: - return False + return self._cache_control_ttl def transform_anthropic_messages_request( self, @@ -114,7 +119,7 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig): """ def __init__(self, provider: SimpleProviderConfig): - super().__init__() + super().__init__(cache_control_ttl=bool(provider.constraints.get("cache_control_ttl"))) self._provider = provider @property @@ -124,9 +129,6 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig): def should_strip_billing_metadata(self) -> bool: return True - def supports_cache_control_ttl(self) -> bool: - return bool(self._provider.constraints.get("cache_control_ttl")) - def _resolve_api_key(self, api_key: str | None) -> str | None: return api_key or get_secret_str(self._provider.api_key_env) or litellm.api_key diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index ad4c3d6bfbb..e819433c269 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -296,21 +296,15 @@ async def test_bedrock_converse_budget_tokens_preserved(): mock_acompletion.assert_called_once() call_kwargs = mock_acompletion.call_args.kwargs - print( - "acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str) - ) + print("acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str)) # Verify thinking parameter is passed through with budget_tokens preserved thinking_param = call_kwargs.get("thinking") - assert ( - thinking_param is not None - ), "thinking parameter should be passed to acompletion" - assert ( - thinking_param.get("type") == "enabled" - ), "thinking.type should be 'enabled'" - assert ( - thinking_param.get("budget_tokens") == 1024 - ), f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" + assert thinking_param is not None, "thinking parameter should be passed to acompletion" + assert thinking_param.get("type") == "enabled", "thinking.type should be 'enabled'" + assert thinking_param.get("budget_tokens") == 1024, ( + f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" + ) def test_openai_model_with_thinking_converts_to_reasoning(): @@ -342,23 +336,18 @@ def test_openai_model_with_thinking_converts_to_reasoning(): call_kwargs = mock_responses.call_args.kwargs # Verify reasoning is set (converted from thinking) - assert ( - "reasoning" in call_kwargs - ), "reasoning should be passed to litellm.responses" + assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses" # budget_tokens=1024 -> effort="low" (at the LOW budget threshold) # reasoning_auto_summary is False by default, so no summary key expected_reasoning = {"effort": "low"} assert call_kwargs["reasoning"] == expected_reasoning, ( - f"reasoning should be {expected_reasoning} for budget_tokens=1024, " - f"got {call_kwargs.get('reasoning')}" + f"reasoning should be {expected_reasoning} for budget_tokens=1024, got {call_kwargs.get('reasoning')}" ) assert "summary" not in call_kwargs["reasoning"] # Verify thinking is NOT passed directly to the Responses API - assert ( - "thinking" not in call_kwargs - ), "thinking should NOT be passed directly to litellm.responses" + assert "thinking" not in call_kwargs, "thinking should NOT be passed directly to litellm.responses" class TestThinkingParameterTransformation: @@ -411,9 +400,7 @@ class TestThinkingParameterTransformation: thinking=thinking, model="openai/gpt-5.2", ) - assert result == { - "reasoning_effort": {"effort": "high", "summary": "detailed"} - } + assert result == {"reasoning_effort": {"effort": "high", "summary": "detailed"}} finally: litellm.reasoning_auto_summary = original @@ -611,9 +598,9 @@ class TestThinkingSummaryPreservation: mock_responses.assert_called_once() call_kwargs = mock_responses.call_args.kwargs reasoning = call_kwargs["reasoning"] - assert ( - reasoning["summary"] == "concise" - ), f"Expected summary='concise', got summary='{reasoning.get('summary')}'" + assert reasoning["summary"] == "concise", ( + f"Expected summary='concise', got summary='{reasoning.get('summary')}'" + ) def test_responses_adapter_preserves_summary(self): """translate_thinking_to_reasoning should include summary when user provides it.""" @@ -622,9 +609,7 @@ class TestThinkingSummaryPreservation: ) thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"} - result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( - thinking - ) + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) assert result == {"effort": "high", "summary": "concise"} def test_responses_adapter_no_summary_by_default(self): @@ -638,11 +623,7 @@ class TestThinkingSummaryPreservation: try: litellm.reasoning_auto_summary = False thinking = {"type": "enabled", "budget_tokens": 5000} - result = ( - LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( - thinking - ) - ) + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) assert result == {"effort": "high"} assert result is not None and "summary" not in result finally: @@ -659,9 +640,7 @@ class TestThinkingSummaryPreservation: thinking=thinking, model="openai/gpt-5.2", ) - assert result == { - "reasoning_effort": {"effort": "high", "summary": "concise"} - } + assert result == {"reasoning_effort": {"effort": "high", "summary": "concise"}} def test_translate_thinking_for_model_disabled_stays_plain_string_when_auto_summary_enabled(self): """Disabled thinking must stay a plain string even when reasoning_auto_summary is on.""" @@ -807,9 +786,7 @@ def test_presanitized_flag_not_leaked_to_provider_params(): def fake_base_handler(*args, **kwargs): captured.update(kwargs) - captured["optional"] = kwargs.get( - "anthropic_messages_optional_request_params", {} - ) + captured["optional"] = kwargs.get("anthropic_messages_optional_request_params", {}) return "stub" with patch.object( @@ -974,6 +951,38 @@ def test_gate_passthrough_skipped_when_only_chat_completions_supported(monkeypat assert "config" not in captured +@pytest.mark.parametrize( + "model_info, expected_ttl_support", + [ + ({"supported_endpoints": ["/v1/messages"]}, False), + ({"supported_endpoints": ["/v1/messages"], "cache_control_ttl": True}, True), + ({"supported_endpoints": ["/v1/messages"], "cache_control_ttl": "yes"}, False), + ], +) +def test_gate_passthrough_forwards_cache_control_ttl_only_when_deployment_opts_in( + monkeypatch, model_info, expected_ttl_support +): + """The passthrough config strips cache_control.ttl unless the deployment sets + model_info.cache_control_ttl to exactly true.""" + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + anthropic_messages_handler, + ) + + captured, _ = _gate_stubs(monkeypatch) + + result = anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "Hello"}], + model="openai/some-model", + api_key="sk-test", + api_base="https://host/v1", + model_info=model_info, + ) + + assert result == "native-passthrough" + assert captured["config"].supports_cache_control_ttl() is expected_ttl_support + + def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag(): """Regional and provider-prefixed Claude 4.8+/5 entries carry ``supports_mid_conversation_system``, but the bare first-party keys @@ -987,9 +996,7 @@ def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_sys import litellm - cost_map_path = os.path.join( - os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json" - ) + cost_map_path = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json") with open(cost_map_path) as f: cost_map = json.load(f) rules = cost_map["fallback_generalizations"]["rules"] @@ -1028,9 +1035,7 @@ def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_sys ("perplexity/sonar", "sonar", "https://api.perplexity.ai/chat/completions"), ], ) -async def test_messages_strips_provider_prefix_exactly_once( - requested_model, expected_wire_model, expected_url -): +async def test_messages_strips_provider_prefix_exactly_once(requested_model, expected_wire_model, expected_url): """ BerriAI/litellm#37716: only the leading provider segment may be stripped on the way upstream. diff --git a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py index d325492914e..e33b03afdff 100644 --- a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py @@ -414,6 +414,23 @@ def test_native_anthropic_config_keeps_cache_control_ttl(): assert payload["system"][0]["cache_control"] == {"type": "ephemeral", "ttl": "5m"} +def test_deployment_opt_in_keeps_cache_control_ttl(): + config = OpenAILikeAnthropicMessagesConfig(cache_control_ttl=True) + payload = config.transform_anthropic_messages_request( + model="some-model", + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + } + ], + anthropic_messages_optional_request_params={"max_tokens": 16}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert payload["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + def test_json_provider_constraint_opts_into_cache_control_ttl(): from litellm.llms.openai_like.json_loader import SimpleProviderConfig from litellm.llms.openai_like.messages.transformation import ( From eac5dc10f338e7faf751dd198d10e0bd7a2c024b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:11:36 -0700 Subject: [PATCH 011/175] fix(guardrails): apply PUT /guardrails/{id} to the serving worker immediately and reject invalid configs with 422 --- .../proxy/guardrails/guardrail_endpoints.py | 20 +++- .../proxy/guardrails/guardrail_registry.py | 37 ++---- .../guardrails/test_guardrail_endpoints.py | 53 +++++++-- .../guardrails/test_guardrail_registry.py | 108 ++++++++++++++---- 4 files changed, 158 insertions(+), 60 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 2b04828f0f2..3d2ed641a30 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -514,12 +514,26 @@ async def update_guardrail( guardrail_name: Final = result.get("guardrail_name", "Unknown") try: - IN_MEMORY_GUARDRAIL_HANDLER.update_in_memory_guardrail( - guardrail_id=guardrail_id, guardrail=cast(Guardrail, result) - ) + IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(guardrail=cast(Guardrail, result)) verbose_proxy_logger.info( "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id ) + except (ValueError, TypeError) as update_error: + # The new config is invalid (a raising guardrail __init__): + # reinitialize_guardrail already restored the previous live instance, but + # update_guardrail_in_db above already persisted the rejected config to + # the DB. Roll that back too, so the DB and the live guardrail never + # disagree about what's actually enforcing, and surface the rejection to + # the caller instead of a misleading 200. + await GUARDRAIL_REGISTRY.update_guardrail_in_db( + guardrail_id=guardrail_id, + guardrail=existing_guardrail, + prisma_client=prisma_client, + ) + raise HTTPException( + status_code=422, + detail=f"Invalid guardrail configuration, update rejected: {update_error}", + ) from update_error except Exception as update_error: verbose_proxy_logger.warning( "Immediate sync: Failed to update '%s' (ID: %s) in memory: %s", diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index dc13c09dd38..60873a1eeb1 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -6,7 +6,7 @@ import os from collections.abc import Callable, Iterator, Mapping from datetime import datetime, timezone from itertools import chain, count -from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, cast +from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol from pydantic import ValidationError @@ -615,26 +615,6 @@ class InMemoryGuardrailHandler: return _guardrail_callback - def update_in_memory_guardrail( - self, - guardrail_id: str, - guardrail: Guardrail, - source: Literal["db", "config"] = "db", - ) -> None: - """ - Update a guardrail in memory - - - updates the guardrail in memory - - updates the guardrail params in litellm.callback_manager - """ - self.IN_MEMORY_GUARDRAILS[guardrail_id] = guardrail - self._sources[guardrail_id] = source - - custom_guardrail_callback: Final = self.guardrail_id_to_custom_guardrail.get(guardrail_id) - if custom_guardrail_callback: - updated_litellm_params: Final = cast(LitellmParams, guardrail.get("litellm_params", {})) - custom_guardrail_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params) - def delete_in_memory_guardrail(self, guardrail_id: str) -> None: """ Delete a guardrail in memory and remove from litellm callbacks. @@ -789,11 +769,12 @@ class InMemoryGuardrailHandler: Removes old callback from litellm.callbacks and creates fresh instance. If the new config fails to initialize (e.g. an invalid on_flagged - combination), the previous instance is restored rather than left - deleted: initialize_guardrail's own ValueError/TypeError propagate - uncaught, so a caller reaching this point after already deleting the - old instance would otherwise leave the guardrail providing no - protection at all, not merely "still enforcing the old config." + combination or an invalid regex), the previous instance is restored + rather than left deleted, and the failure is re-raised as ValueError so + every init failure reaches callers as one exception type: a caller + reaching this point after already deleting the old instance would + otherwise leave the guardrail providing no protection at all, not + merely "still enforcing the old config." """ guardrail_id: Final = guardrail.get("guardrail_id") if not guardrail_id: @@ -812,7 +793,7 @@ class InMemoryGuardrailHandler: # that was enforcing must never fail open because an update was bad. try: return self.initialize_guardrail(guardrail=guardrail, config_file_path=config_file_path, source=source) - except Exception: + except Exception as init_error: if previous_guardrail is not None: verbose_proxy_logger.exception( "Reinitializing guardrail %s with updated params failed; restoring the previous configuration", @@ -824,7 +805,7 @@ class InMemoryGuardrailHandler: ) except Exception: # noqa: BLE001 # the original failure must propagate even if the restore breaks verbose_proxy_logger.exception("Restoring previous guardrail %s also failed", guardrail_id) - raise + raise ValueError(f"Guardrail initialization failed: {init_error}") from init_error def sync_guardrail_from_db(self, guardrail: Guardrail, config_file_path: str | None = None) -> Guardrail | None: """ diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index 9b2117b7647..320e51203f6 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -104,7 +104,7 @@ def mock_in_memory_handler(mocker): mock_handler.get_guardrail_by_id.return_value = MOCK_CONFIG_GUARDRAIL mock_handler.get_source.return_value = "config" mock_handler.initialize_guardrail = mocker.Mock() - mock_handler.update_in_memory_guardrail = mocker.Mock() + mock_handler.sync_guardrail_from_db = mocker.Mock() mock_handler.delete_in_memory_guardrail = mocker.Mock() mock_handler.reconcile_db_guardrails = mocker.Mock(return_value=[]) return mock_handler @@ -1045,13 +1045,15 @@ async def test_create_guardrail_endpoint( "scenario,expected_result,expected_exception", [ ("success_with_sync", "test-db-guardrail", None), - ("success_sync_fails", "test-db-guardrail", None), + ("success_sync_fails_unexpected_error", "test-db-guardrail", None), + ("sync_fails_invalid_config", None, HTTPException), ("database_failure", None, HTTPException), ("no_prisma_client", None, HTTPException), ], ids=[ "success_with_immediate_sync", - "success_but_sync_fails", + "success_but_sync_fails_with_unexpected_error", + "sync_rejects_invalid_config", "database_error", "missing_prisma_client", ], @@ -1071,6 +1073,7 @@ async def test_update_guardrail_endpoint( mock_logger = None if scenario == "success_with_sync": mock_prisma_client = mocker.Mock() + mock_in_memory_handler.sync_guardrail_from_db = mocker.Mock() mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch( "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", @@ -1081,10 +1084,13 @@ async def test_update_guardrail_endpoint( mock_in_memory_handler, ) - elif scenario == "success_sync_fails": + elif scenario == "success_sync_fails_unexpected_error": + # A non-ValueError/TypeError failure is not a config-rejection signal, + # so it keeps the pre-existing swallow-and-warn behavior rather than + # rolling back the DB write. mock_prisma_client = mocker.Mock() - mock_in_memory_handler.update_in_memory_guardrail.side_effect = Exception( - "Sync failed" + mock_in_memory_handler.sync_guardrail_from_db = mocker.Mock( + side_effect=Exception("Sync failed") ) mock_logger = mocker.patch( "litellm.proxy.guardrails.guardrail_endpoints.verbose_proxy_logger" @@ -1100,6 +1106,25 @@ async def test_update_guardrail_endpoint( mock_in_memory_handler, ) + elif scenario == "sync_fails_invalid_config": + # Regression for the PUT half of the fix: a TypeError from the sync (the + # deleted update_in_memory_guardrail raised exactly this on every PUT) + # must roll back the DB write and surface a 422, not persist the + # rejected config with a 200. + mock_prisma_client = mocker.Mock() + mock_in_memory_handler.sync_guardrail_from_db = mocker.Mock( + side_effect=TypeError("vars() argument must have __dict__ attribute") + ) + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) # test-quality-ok: reused pattern + mocker.patch( # test-quality-ok: reused pattern + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", + mock_guardrail_registry, + ) + mocker.patch( # test-quality-ok: reused pattern + "litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", + mock_in_memory_handler, + ) + elif scenario == "database_failure": mock_prisma_client = mocker.Mock() mock_guardrail_registry.update_guardrail_in_db.side_effect = Exception( @@ -1128,6 +1153,16 @@ async def test_update_guardrail_endpoint( assert "Database error" in str(exc_info.value.detail) elif scenario == "no_prisma_client": assert "Prisma client not initialized" in str(exc_info.value.detail) + elif scenario == "sync_fails_invalid_config": + assert exc_info.value.status_code == 422 + assert "update rejected" in str(exc_info.value.detail) + # Rolled back: update_guardrail_in_db is called once for the + # rejected write and once more to restore the previous config. + assert mock_guardrail_registry.update_guardrail_in_db.call_count == 2 + assert ( + mock_guardrail_registry.update_guardrail_in_db.call_args.kwargs["guardrail"] + == MOCK_DB_GUARDRAIL + ) else: result = await update_guardrail( @@ -1143,11 +1178,11 @@ async def test_update_guardrail_endpoint( prisma_client=mocker.ANY, ) - mock_in_memory_handler.update_in_memory_guardrail.assert_called_once_with( - guardrail_id="test-guardrail-id", guardrail=mocker.ANY + mock_in_memory_handler.sync_guardrail_from_db.assert_called_once_with( + guardrail=mocker.ANY ) - if scenario == "success_sync_fails": + if scenario == "success_sync_fails_unexpected_error": assert mock_logger is not None mock_logger.warning.assert_called_once() assert "Failed to update" in str(mock_logger.warning.call_args) diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 2c0735970d3..beaffa73100 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -154,29 +154,51 @@ def test_duplicate_config_guardrail_names_get_distinct_stable_ids(): registry_module.guardrail_initializer_registry.pop("dup_name_test", None) -def test_update_in_memory_guardrail(): +def test_sync_guardrail_from_db_applies_db_dict_params_to_live_instance(): + """ + Regression for PUT /guardrails/{id}: the DB row arrives with litellm_params as + a plain jsonb dict, and the deleted update_in_memory_guardrail cast it to + LitellmParams without constructing one, so vars() raised and the running proxy + kept enforcing the stale config forever. The PUT endpoint now routes through + sync_guardrail_from_db, which must rebuild the live instance from the dict: + new blocked words compiled in, old ones gone, and the event hook re-derived + from mode (the base-class setattr path wrote self.mode while dispatch reads + self.event_hook, so only a full re-init applies a mode change). + """ + from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, + ) + handler = InMemoryGuardrailHandler() - handler.guardrail_id_to_custom_guardrail["123"] = CustomGuardrail( - guardrail_name="test-guardrail", - default_on=False, - event_hook=GuardrailEventHooks.pre_call, - ) + gid = "66666666-6666-6666-6666-666666666666" - handler.update_in_memory_guardrail( - "123", - Guardrail( - guardrail_name="test-guardrail", - litellm_params=LitellmParams(guardrail="test-guardrail", mode="pre_call", default_on=True), - ), - ) - - assert ( - handler.guardrail_id_to_custom_guardrail["123"].should_run_guardrail( - data={}, event_type=GuardrailEventHooks.pre_call + def db_guardrail(word: str, mode: str) -> Guardrail: + return Guardrail( + guardrail_id=gid, + guardrail_name="cf-put-sync", + litellm_params={ + "guardrail": "litellm_content_filter", + "mode": mode, + "default_on": True, + "blocked_words": [{"keyword": word, "action": "BLOCK"}], + }, ) - is True - ) - assert handler.guardrail_id_to_custom_guardrail["123"].event_hook is GuardrailEventHooks.pre_call + + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + handler.sync_guardrail_from_db(db_guardrail("foobarblock", "pre_call")) + handler.sync_guardrail_from_db(db_guardrail("quxnewblock", "during_call")) + + instance = handler.guardrail_id_to_custom_guardrail[gid] + assert isinstance(instance, ContentFilterGuardrail) + assert instance._check_blocked_words("hello QUXNEWBLOCK") is not None + assert instance._check_blocked_words("hello FOOBARBLOCK") is None + assert instance.event_hook == GuardrailEventHooks.during_call + assert instance.should_run_guardrail(data={}, event_type=GuardrailEventHooks.during_call) is True + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot def _make_guardrail(guardrail_id: str, name: str = "g") -> Guardrail: @@ -774,3 +796,49 @@ def test_reinitialize_guardrail_restores_previous_on_failure(): assert restored.guardrail_name == "restore-me" finally: registry_module.guardrail_initializer_registry.pop("restore_test", None) + + +def test_reinitialize_guardrail_raises_value_error_for_non_value_error_init_failures(): + """Regression for the LIT-6479 fix's 422 path: a constructor failure that is not + already a ValueError/TypeError (re.error from an invalid regex has neither in its + MRO) must still surface as ValueError, so the PUT/PATCH endpoints' rollback+422 + catch is exhaustive instead of warn-and-200 persisting a broken config.""" + import re + + from litellm.proxy.guardrails import guardrail_registry as registry_module + + def _initializer(litellm_params, guardrail): + if litellm_params.api_key == "bad-regex": + re.compile("([") + return CustomGuardrail( + guardrail_name=guardrail["guardrail_name"], + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + + registry_module.guardrail_initializer_registry["regex_test"] = _initializer + try: + handler = InMemoryGuardrailHandler() + created = handler.initialize_guardrail( + guardrail={ + "guardrail_name": "regex-me", + "litellm_params": {"guardrail": "regex_test", "mode": "pre_call", "api_key": "ok"}, + }, + ) + guardrail_id = created["guardrail_id"] + + with pytest.raises(ValueError, match="Guardrail initialization failed") as excinfo: + handler.reinitialize_guardrail( + guardrail={ + "guardrail_id": guardrail_id, + "guardrail_name": "regex-me", + "litellm_params": {"guardrail": "regex_test", "mode": "pre_call", "api_key": "bad-regex"}, + }, + ) + + assert isinstance(excinfo.value.__cause__, re.error) + assert guardrail_id in handler.IN_MEMORY_GUARDRAILS + restored = handler.guardrail_id_to_custom_guardrail[guardrail_id] + assert restored is not None and restored.guardrail_name == "regex-me" + finally: + registry_module.guardrail_initializer_registry.pop("regex_test", None) From 9e25dd708fa7a8b2af354e6901683fd604c4a19a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:47:13 -0700 Subject: [PATCH 012/175] feat(streaming): carry final response cost on streamed usage by default Streamed responses through the proxy previously exposed no usable cost: the x-litellm-response-cost header is unreadable mid-stream and the final usage chunk carried only tokens, priced against an alias model name the client cannot resolve. The include_cost_in_streaming_usage flag existed but was off by default and only fixed the wire, not SDK clients. Stamp usage.cost into the joined streaming response by default wherever a final usage object is built: the chat-completions stream_chunk_builder, the native /v1/responses RESPONSE_COMPLETED event, and synthetic response events. Provider-reported cost always wins over the computed value, and only positive computed costs are stamped so unpriceable alias responses keep deferring to the logging object's own calculation. Per-chunk SSE cost injection (/v1/messages, generateContent, passthrough) stays behind the flag. Also normalize non-litellm usage objects in stream_chunk_builder: openai CompletionUsage lacks Usage.__contains__, so membership probes silently returned False and client-side rebuilds dropped the wire cost and recounted token usage locally. Wire token counts and cost now survive. Resolves LIT-6427 --- .../streaming_chunk_builder_utils.py | 8 ++- litellm/main.py | 22 ++++--- .../streaming_iterator.py | 10 --- litellm/responses/streaming_iterator.py | 46 ++++++------- .../test_streaming_chunk_builder_utils.py | 53 +++++++++++++++ .../responses/test_streaming_iterator.py | 52 +++++++++++++++ tests/test_litellm/test_main.py | 66 +++++++++++++++++-- 7 files changed, 204 insertions(+), 53 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 0e2139d688b..276616f9eee 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -36,6 +36,8 @@ from litellm.types.utils import ( from litellm.utils import print_verbose, token_counter if TYPE_CHECKING: + from openai.types.completion_usage import CompletionUsage + from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.litellm_core_utils.streaming_chunk_builder_utils import ( UsagePerChunk, @@ -782,7 +784,7 @@ class ChunkProcessor: @staticmethod def _extract_usage_chunk(chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream") -> Usage | None: - usage_chunk: Usage | None = None + usage_chunk: Usage | CompletionUsage | None = None if hasattr(chunk, "usage") and chunk.usage is not None: usage_chunk = chunk.usage elif "usage" in chunk: @@ -794,7 +796,9 @@ class ChunkProcessor: if isinstance(usage_chunk, dict): return Usage(**usage_chunk) - return usage_chunk + if usage_chunk is None or isinstance(usage_chunk, Usage): + return usage_chunk + return Usage(**usage_chunk.model_dump()) def _calculate_usage_per_chunk( self, diff --git a/litellm/main.py b/litellm/main.py index 0c8bff16f81..7ca84226b09 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8634,6 +8634,16 @@ def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Opti hidden_params["response_cost"] = response_cost +def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: + if logging_obj is None: + return + if isinstance(getattr(usage, "cost", None), (int, float)): + return + computed_cost: Final = logging_obj._response_cost_calculator(result=response) + if isinstance(computed_cost, (int, float)) and computed_cost > 0: + setattr(usage, "cost", computed_cost) + + def stream_chunk_builder( chunks: list, messages: list | None = None, @@ -8728,12 +8738,7 @@ def stream_chunk_builder( ) break - if litellm.include_cost_in_streaming_usage and logging_obj is not None: - setattr( - usage, - "cost", - logging_obj._response_cost_calculator(result=response), - ) + _stamp_streaming_usage_cost(usage, response, logging_obj) _set_stream_builder_response_cost(response, logging_obj) processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) @@ -8912,10 +8917,7 @@ def stream_chunk_builder( ) break - # Add cost to usage object if include_cost_in_streaming_usage is True - if litellm.include_cost_in_streaming_usage and logging_obj is not None: - setattr(usage, "cost", logging_obj._response_cost_calculator(result=response)) - + _stamp_streaming_usage_cost(usage, response, logging_obj) _set_stream_builder_response_cost(response, logging_obj) processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 8b1eeb30306..27afff39c0f 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1164,16 +1164,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _emit_response_completed_event(self, litellm_model_response: ModelResponse) -> ResponseCompletedEvent | None: if litellm_model_response: - # Add cost to usage object if include_cost_in_streaming_usage is True - if litellm.include_cost_in_streaming_usage and self.litellm_logging_obj is not None: - usage: Final[object] = getattr(litellm_model_response, "usage", None) - if usage is not None: - setattr( - usage, - "cost", - self.litellm_logging_obj._response_cost_calculator(result=litellm_model_response), - ) - # Transform the response responses_api_response: Final = ( LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index d070f7758fd..2b4252aa1c5 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -405,23 +405,7 @@ class BaseResponsesAPIStreamingIterator: openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED, ): self.completed_response = openai_responses_api_chunk - # Add cost to usage object if include_cost_in_streaming_usage is True - if litellm.include_cost_in_streaming_usage and self.logging_obj is not None: - response_obj: Final[ResponsesAPIResponse | None] = getattr( - openai_responses_api_chunk, "response", None - ) - if response_obj: - usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None) - if usage_obj is not None: - try: - cost: Final[float | None] = self.logging_obj._response_cost_calculator( - result=response_obj - ) - if cost is not None: - setattr(usage_obj, "cost", cost) - except Exception: - # Best-effort usage cost annotation should not break stream replay. - pass + _stamp_responses_usage_cost(getattr(openai_responses_api_chunk, "response", None), self.logging_obj) if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED: self._handle_logging_failed_response() @@ -1272,6 +1256,24 @@ def _add_text_like_part_events( ) +def _stamp_responses_usage_cost( + response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None +) -> None: + if response_obj is None or logging_obj is None: + return + usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None) + if usage_obj is None: + return + if isinstance(getattr(usage_obj, "cost", None), (int, float)): + return + try: + cost: Final[float | None] = logging_obj._response_cost_calculator(result=response_obj) + except Exception: + return + if isinstance(cost, (int, float)) and cost > 0: + setattr(usage_obj, "cost", cost) + + def _build_synthetic_response_events( *, transformed: ResponsesAPIResponse, @@ -1279,15 +1281,7 @@ def _build_synthetic_response_events( chunk_size: int, ) -> list[ResponsesAPIStreamingResponse]: openai_types: Final = _get_openai_response_types() - if litellm.include_cost_in_streaming_usage and logging_obj is not None: - usage_obj: Final = transformed.usage if hasattr(transformed, "usage") else None - if usage_obj is not None: - try: - cost: Final[float | None] = logging_obj._response_cost_calculator(result=transformed) - if cost is not None: - setattr(usage_obj, "cost", cost) - except Exception: - pass + _stamp_responses_usage_cost(transformed, logging_obj) events: Final[list[ResponsesAPIStreamingResponse]] = [ _build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed), diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 8ac050a04f9..bacbcbf132b 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -592,6 +592,59 @@ def test_stream_chunk_builder_litellm_usage_chunks(): assert usage.total_tokens == 77 +def test_calculate_usage_honors_openai_sdk_completion_usage_chunks(): + from openai.types.completion_usage import CompletionUsage + + content_chunk = ModelResponseStream( + id="chatcmpl-sdk-usage-1", + created=1745513206, + model="mantle-claude", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta( + provider_specific_fields=None, + content="ok", + role=None, + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields=None, + stream_options={"include_usage": True}, + ) + usage_chunk = ModelResponseStream( + id="chatcmpl-sdk-usage-1", + created=1745513207, + model="mantle-claude", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[], + provider_specific_fields=None, + stream_options={"include_usage": True}, + ) + usage_chunk.usage = CompletionUsage( + prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704 + ) + assert type(usage_chunk.usage) is CompletionUsage + + chunks = [content_chunk, usage_chunk] + usage = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, model="mantle-claude", completion_output="" + ) + + assert usage.prompt_tokens == 20 + assert usage.completion_tokens == 60 + assert usage.total_tokens == 80 + assert getattr(usage, "cost", None) == pytest.approx(0.000704) + + def test_get_model_from_chunks_azure_model_router(): """ Test that _get_model_from_chunks finds the actual model from Azure Model Router chunks. diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index 677faf7f655..9edcaaef034 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -326,3 +326,55 @@ def test_run_post_success_hooks_does_not_report_generation_time_as_overhead(): assert iterator.completed_response._hidden_params["_response_ms"] == 10000.0 assert "litellm_overhead_time_ms" not in iterator.completed_response._hidden_params + + +def _responses_api_response_with_usage() -> ResponsesAPIResponse: + from litellm.types.llms.openai import ResponseAPIUsage + + return ResponsesAPIResponse( + id="resp_lit6427", + created_at=int(datetime(2025, 1, 1).timestamp()), + status="completed", + model="mantle-claude", + object="response", + output=[], + usage=ResponseAPIUsage(input_tokens=20, output_tokens=60, total_tokens=80), + ) + + +def test_stamp_responses_usage_cost_stamps_computed_cost(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + + response = _responses_api_response_with_usage() + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj._response_cost_calculator.return_value = 0.000704 + + _stamp_responses_usage_cost(response, logging_obj) + + assert getattr(response.usage, "cost", None) == pytest.approx(0.000704) + logging_obj._response_cost_calculator.assert_called_once_with(result=response) + + +def test_stamp_responses_usage_cost_keeps_provider_reported_cost(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + + response = _responses_api_response_with_usage() + setattr(response.usage, "cost", 0.5) + logging_obj = Mock(spec=LiteLLMLoggingObj) + + _stamp_responses_usage_cost(response, logging_obj) + + assert getattr(response.usage, "cost", None) == pytest.approx(0.5) + logging_obj._response_cost_calculator.assert_not_called() + + +def test_stamp_responses_usage_cost_survives_calculator_failure(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + + response = _responses_api_response_with_usage() + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj._response_cost_calculator.side_effect = RuntimeError("cost map unavailable") + + _stamp_responses_usage_cost(response, logging_obj) + + assert getattr(response.usage, "cost", None) is None diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 8cf878d05d9..7c2b9d0be05 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -3150,8 +3150,8 @@ def _stream_builder_logging_obj() -> LiteLLMLogging: return logging_obj -def test_stream_chunk_builder_reports_streaming_usage_cost_when_enabled(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) +def test_stream_chunk_builder_stamps_streaming_usage_cost_by_default(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) chunks: Final = [ _stream_builder_text_chunk("gpt-4o", "Hello "), _stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"), @@ -3168,11 +3168,45 @@ def test_stream_chunk_builder_reports_streaming_usage_cost_when_enabled(monkeypa assert response._hidden_params["response_cost"] == pytest.approx(usage_cost) -def test_stream_chunk_builder_defers_cost_to_logging_obj_when_usage_cost_absent(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) +def test_stream_chunk_builder_skips_stamp_when_cost_is_unpriceable(): + import time as time_module + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + logging_obj: Final = LiteLLMLogging( + model="us.anthropic.claude-opus-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="completion", + start_time=time_module.time(), + litellm_call_id="stream-builder-alias-unpriceable", + function_id="1", + ) + logging_obj.model_call_details["custom_llm_provider"] = "bedrock" + logging_obj.optional_params = {} + usage_chunk: Final = _stream_builder_text_chunk("bedrock-claude-opus-5", "") + usage_chunk.usage = Usage(prompt_tokens=40, completion_tokens=5, total_tokens=45) + chunks: Final = [ + _stream_builder_text_chunk("bedrock-claude-opus-5", "Hello ", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=logging_obj + ) + + assert response is not None + assert getattr(response.usage, "cost", None) is None + assert response._hidden_params.get("response_cost") is None + + +def test_stream_chunk_builder_keeps_provider_reported_usage_cost(): + usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "") + usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5) chunks: Final = [ _stream_builder_text_chunk("gpt-4o", "Hello "), _stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"), + usage_chunk, ] response: Final = litellm.stream_chunk_builder( @@ -3180,4 +3214,26 @@ def test_stream_chunk_builder_defers_cost_to_logging_obj_when_usage_cost_absent( ) assert response is not None - assert response._hidden_params.get("response_cost") is None + assert getattr(response.usage, "cost", None) == pytest.approx(0.5) + assert response._hidden_params["response_cost"] == pytest.approx(0.5) + + +def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk(): + from openai.types.completion_usage import CompletionUsage + + usage_chunk: Final = _stream_builder_text_chunk("mantle-claude", "") + usage_chunk.usage = CompletionUsage(prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704) + assert type(usage_chunk.usage) is CompletionUsage + chunks: Final = [ + _stream_builder_text_chunk("mantle-claude", "Hello "), + _stream_builder_text_chunk("mantle-claude", "world.", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder(chunks=chunks, messages=[{"role": "user", "content": "hi"}]) + + assert response is not None + assert response.usage.prompt_tokens == 20 + assert response.usage.completion_tokens == 60 + assert getattr(response.usage, "cost", None) == pytest.approx(0.000704) + assert response._hidden_params["response_cost"] == pytest.approx(0.000704) From d4fc54a11d1ac18f10c33741b760b99716faac93 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 08:25:42 +0000 Subject: [PATCH 013/175] chore(techdebt): clear fresh debt from the 2026-08-31 window Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- basedpyright-code-budget.json | 6 +-- litellm/llms/gigachat/authenticator.py | 28 +++++--------- litellm/llms/gigachat/chat/streaming.py | 1 - litellm/llms/gigachat/chat/transformation.py | 38 +++++-------------- .../llms/gigachat/embedding/transformation.py | 14 ++----- .../gigachat/passthrough/transformation.py | 2 - litellm/llms/gigachat/utils.py | 1 - litellm/passthrough/main.py | 3 -- .../llm_passthrough_endpoints.py | 4 +- .../router_strategy/test_complexity_router.py | 6 ++- type-discipline-budget.json | 8 ++-- 11 files changed, 35 insertions(+), 76 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index a07b9352659..d84aacfeaf0 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5607 }, "reportMissingTypeArgument": { - "limit": 15310 + "limit": 15308 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,13 +105,13 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38368 + "limit": 38367 }, "reportUnknownParameterType": { "limit": 19633 }, "reportUnknownVariableType": { - "limit": 29908 + "limit": 29906 }, "reportUnnecessaryCast": { "limit": 111 diff --git a/litellm/llms/gigachat/authenticator.py b/litellm/llms/gigachat/authenticator.py index d6b217d5746..73086ba395b 100644 --- a/litellm/llms/gigachat/authenticator.py +++ b/litellm/llms/gigachat/authenticator.py @@ -8,6 +8,7 @@ Based on official GigaChat SDK authentication flow. import time import uuid from collections.abc import Mapping +from types import MappingProxyType from typing import Final import httpx @@ -32,8 +33,8 @@ GIGACHAT_SCOPE: Final = "GIGACHAT_API_PERS" # Token expiry buffer in milliseconds (refresh token 60s before expiry) TOKEN_EXPIRY_BUFFER_MS: Final = 60000 -# Cache for access tokens _token_cache: Final = InMemoryCache() +_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) class GigaChatAuthError(BaseLLMException): @@ -80,10 +81,9 @@ def get_access_token( Raises: GigaChatAuthError: If authentication fails """ - if not litellm_params: - litellm_params = {} # mutable-ok: empty dict default; rebind-ok: provide default + params: Final = litellm_params or _NO_LITELLM_PARAMS - access_token: Final = litellm_params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN") + access_token: Final = params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN") if access_token: return access_token @@ -94,24 +94,20 @@ def get_access_token( message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.", ) - effective_scope: Final = scope or litellm_params.get("gigachat_scope") or _get_scope() - effective_auth_url: Final = auth_url or litellm_params.get("gigachat_auth_url") or _get_auth_url() + effective_scope: Final = scope or params.get("gigachat_scope") or _get_scope() + effective_auth_url: Final = auth_url or params.get("gigachat_auth_url") or _get_auth_url() - # Check cache cache_key: Final = f"gigachat_token:{effective_credentials[:16]}" cached: Final = _token_cache.get_cache(cache_key) if cached: _token, _expires_at = cached - # Check if token is still valid (with buffer) if time.time() * 1000 < _expires_at - TOKEN_EXPIRY_BUFFER_MS: verbose_logger.debug("Using cached GigaChat access token") return _token - # Request new token new_token, new_expires_at = _request_token_sync(effective_credentials, effective_scope, effective_auth_url) # pyright: ignore[reportArgumentType] # credential keys may be broader than str if new_expires_at: - # Cache token ttl_seconds: Final = max(0, (new_expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) if ttl_seconds > 0: _token_cache.set_cache(cache_key, (new_token, new_expires_at), ttl=ttl_seconds) @@ -126,10 +122,9 @@ async def get_access_token_async( litellm_params: Mapping[str, object] | None = None, ) -> str: """Async version of get_access_token.""" - if not litellm_params: - litellm_params = {} # mutable-ok: empty dict default; rebind-ok: provide default + params: Final = litellm_params or _NO_LITELLM_PARAMS - access_token: Final = litellm_params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN") + access_token: Final = params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN") if access_token: return access_token @@ -140,10 +135,9 @@ async def get_access_token_async( message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.", ) - effective_scope: Final = scope or litellm_params.get("gigachat_scope") or _get_scope() - effective_auth_url: Final = auth_url or litellm_params.get("gigachat_auth_url") or _get_auth_url() + effective_scope: Final = scope or params.get("gigachat_scope") or _get_scope() + effective_auth_url: Final = auth_url or params.get("gigachat_auth_url") or _get_auth_url() - # Check cache cache_key: Final = f"gigachat_token:{effective_credentials[:16]}" cached: Final = _token_cache.get_cache(cache_key) if cached: @@ -152,11 +146,9 @@ async def get_access_token_async( verbose_logger.debug("Using cached GigaChat access token") return _token - # Request new token new_token, new_expires_at = await _request_token_async(effective_credentials, effective_scope, effective_auth_url) # pyright: ignore[reportArgumentType] # credential keys may be broader than str if new_expires_at: - # Cache token ttl_seconds: Final = max(0, (new_expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) if ttl_seconds > 0: _token_cache.set_cache(cache_key, (new_token, new_expires_at), ttl=ttl_seconds) diff --git a/litellm/llms/gigachat/chat/streaming.py b/litellm/llms/gigachat/chat/streaming.py index 2875b30232e..0a4cbd8e520 100644 --- a/litellm/llms/gigachat/chat/streaming.py +++ b/litellm/llms/gigachat/chat/streaming.py @@ -52,7 +52,6 @@ class GigaChatModelResponseIterator: tool_use: ChatCompletionToolCallChunk | None = None # rebind-ok: conditionally assigned on function_call finish_reason: str | None = chunk_finish_reason - # Handle function_call in stream raw_function_call: Final = delta.get("function_call") if chunk_finish_reason == "function_call" and isinstance(raw_function_call, Mapping) and raw_function_call: func_call: Final[Mapping[str, object]] = raw_function_call diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 8f23c5175ec..991a93ccb21 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -111,11 +111,9 @@ class GigaChatConfig(BaseConfig): """ Set up headers with OAuth token. """ - # Get access token credentials: Final = api_key or get_secret_str("GIGACHAT_CREDENTIALS") or get_secret_str("GIGACHAT_API_KEY") access_token: Final = get_access_token(credentials=credentials, litellm_params=litellm_params) - # Store credentials for image uploads self._current_credentials = credentials self._current_api_base = api_base @@ -208,18 +206,16 @@ class GigaChatConfig(BaseConfig): def _convert_tools_to_functions(self, tools: Sequence) -> Sequence[dict]: """Convert OpenAI tools format to GigaChat functions format.""" - functions: Final[list[dict]] = [] # mutable-ok: accumulator for building functions list - for tool in tools: - if isinstance(tool, dict) and tool.get("type") == "function": - func = tool.get("function", {}) - functions.append( - { - "name": func.get("name", ""), - "description": func.get("description", ""), - "parameters": func.get("parameters", {}), - } - ) - return functions + return [ + { + "name": function.get("name", ""), + "description": function.get("description", ""), + "parameters": function.get("parameters", {}), + } + for function in ( + tool.get("function", {}) for tool in tools if isinstance(tool, dict) and tool.get("type") == "function" + ) + ] def _map_tool_choice(self, tool_choice: str | Mapping[str, object]) -> str | Mapping[str, object] | None: """ @@ -299,7 +295,6 @@ class GigaChatConfig(BaseConfig): if part.get("type") == "text": texts.append(part.get("text", "")) elif part.get("type") == "image_url": - # Extract image URL and upload to GigaChat image_url: object = part.get("image_url", {}) upload_url: str if isinstance(image_url, str): @@ -322,16 +317,13 @@ class GigaChatConfig(BaseConfig): headers: Mapping[str, object], ) -> dict: # mutable-ok: request payload sent to httpx """Transform OpenAI request to GigaChat format.""" - # Transform messages giga_messages: Final = self._transform_messages(messages) - # Build request request_data: Final[dict[str, object]] = { "model": model.replace("gigachat/", ""), "messages": giga_messages, } - # Add optional params for key in [ "temperature", "top_p", @@ -343,7 +335,6 @@ class GigaChatConfig(BaseConfig): if key in optional_params: request_data[key] = optional_params[key] - # Add functions if present if "functions" in optional_params: request_data["functions"] = optional_params["functions"] if "function_call" in optional_params: @@ -358,10 +349,8 @@ class GigaChatConfig(BaseConfig): for i, msg in enumerate(messages): message = dict(msg) - # Remove unsupported fields message.pop("name", None) - # Transform roles role = message.get("role", "user") if role == "developer": message["role"] = "system" @@ -374,18 +363,15 @@ class GigaChatConfig(BaseConfig): if not isinstance(content, str) or not is_valid_json(content): message["content"] = json.dumps(content, ensure_ascii=False) - # Handle None content if message.get("content") is None: message["content"] = "" - # Handle list content (multimodal) - extract text and images content = message.get("content") if isinstance(content, list): message["content"], attachments = self._transform_list_content(content) if attachments: message["attachments"] = attachments - # Transform tool_calls to function_call tool_calls = message.get("tool_calls") if tool_calls and isinstance(tool_calls, list) and len(tool_calls) > 0: tool_call = tool_calls[0] @@ -436,13 +422,11 @@ class GigaChatConfig(BaseConfig): message_data = choice.get("message", {}) finish_reason = choice.get("finish_reason", "stop") - # Transform function_call to tool_calls or content if finish_reason == "function_call" and message_data.get("function_call"): func_call = message_data["function_call"] args = func_call.get("arguments", {}) if is_structured_output: - # Convert to content for structured output if isinstance(args, dict): content = json.dumps(args, ensure_ascii=False) else: @@ -452,7 +436,6 @@ class GigaChatConfig(BaseConfig): message_data.pop("functions_state_id", None) finish_reason = "stop" else: - # Convert to tool_calls format if isinstance(args, dict): args = json.dumps(args, ensure_ascii=False) message_data["tool_calls"] = [ @@ -468,7 +451,6 @@ class GigaChatConfig(BaseConfig): message_data.pop("function_call", None) finish_reason = "tool_calls" - # Clean up GigaChat-specific fields message_data.pop("functions_state_id", None) choices.append( diff --git a/litellm/llms/gigachat/embedding/transformation.py b/litellm/llms/gigachat/embedding/transformation.py index 2ec8324e33c..0db4475be8f 100644 --- a/litellm/llms/gigachat/embedding/transformation.py +++ b/litellm/llms/gigachat/embedding/transformation.py @@ -112,18 +112,10 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig): "input": ["text1", "text2", ...] } """ - # Normalize input to list - if isinstance(input, str): - input_list: list = [input] # rebind-ok: locally scoped conversion - else: - input_list = input - - # Remove gigachat/ prefix from model if present - model = model.removeprefix("gigachat/") # rebind-ok: parameter reassignment for normalization - + normalized_input: Final = [input] if isinstance(input, str) else input # mutable-ok: preserve list API return { - "model": model, - "input": input_list, + "model": model.removeprefix("gigachat/"), + "input": normalized_input, } def transform_embedding_response( diff --git a/litellm/llms/gigachat/passthrough/transformation.py b/litellm/llms/gigachat/passthrough/transformation.py index a0edc6f5682..e1f73d04275 100644 --- a/litellm/llms/gigachat/passthrough/transformation.py +++ b/litellm/llms/gigachat/passthrough/transformation.py @@ -60,7 +60,6 @@ class GigaChatPassthroughConfig(BasePassthroughConfig): """ Set up headers with OAuth token. """ - # Get access token access_token: Final = get_access_token(credentials=api_key, litellm_params=litellm_params) headers["Authorization"] = f"Bearer {access_token}" # rebind-ok: mutating for OAuth setup @@ -82,7 +81,6 @@ class GigaChatPassthroughConfig(BasePassthroughConfig): from litellm.types.utils import LlmProviders, ModelResponse from litellm.utils import ProviderConfigManager - # cost tracking only for completions and embeddings if "completions" in endpoint: provider_chat_config: Final = ProviderConfigManager.get_provider_chat_config( provider=LlmProviders(custom_llm_provider), diff --git a/litellm/llms/gigachat/utils.py b/litellm/llms/gigachat/utils.py index cbb35cd1b57..ce7e848ed7f 100644 --- a/litellm/llms/gigachat/utils.py +++ b/litellm/llms/gigachat/utils.py @@ -4,7 +4,6 @@ from typing import Final from litellm.secret_managers.main import get_secret_str from litellm.types.utils import PromptTokensDetailsWrapper, Usage -# GigaChat API endpoint GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1" diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index 9095cee15a9..689c34b7a88 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -113,10 +113,8 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[Any, Any]): ) ) - # Compliant: Save a strong reference to prevent GC self._background_tasks.add(task) - # Remove the task from the set when it finishes to avoid memory leaks task.add_done_callback(self._background_tasks.discard) except Exception as e: # noqa: BLE001 # Safe catch-all for verbose logging verbose_logger.exception( @@ -578,7 +576,6 @@ def llm_passthrough_route( else: return response except Exception as e: - # provider_config is guaranteed non-None here due to the earlier guard assert provider_config is not None raise base_llm_http_handler._handle_error( e=e, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 78d8ce296b8..b48b8d81494 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1731,7 +1731,7 @@ def get_vertex_ai_allowed_incoming_headers(request: Request) -> dict: def get_vertex_pass_through_handler( - call_type: Literal["discovery", "aiplatform"], # noqa: UP037 + call_type: Literal["discovery", "aiplatform"], # noqa: UP037 # ruff reports quoted Literal values here ) -> BaseVertexAIPassThroughHandler: if call_type == "discovery": return VertexAIDiscoveryPassThroughHandler() @@ -2961,7 +2961,6 @@ async def handle_gigachat_passthrough_router_model( """ from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing - # Detect streaming based on request body is_streaming: Final = request_body.get("stream", False) # pyright: ignore[reportUnknownVariableType] # request_body is dict[Unknown, Unknown] data: dict[str, Any] = await _read_request_body( @@ -2997,7 +2996,6 @@ async def handle_gigachat_passthrough_router_model( data["json"] = request_body data["custom_llm_provider"] = "gigachat" - # Remove sensitive keys from data keys: Final = [ # mutable-ok: list of keys to remove from data "gigachat_auth_url", "gigachat_access_token", diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 1ec8be88c9b..93803ce1005 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -10268,7 +10268,8 @@ class TestContextWindowEscalation: litellm_router_instance=_windowed_router(_SMALL, _BIG), complexity_router_config=_tier_config(session_affinity=True), ) - session_kwargs = lambda: {"metadata": {"session_id": "s-1", "user_api_key_hash": "k-1"}} # noqa: E731 + def session_kwargs() -> dict[str, object]: + return {"metadata": {"session_id": "s-1", "user_api_key_hash": "k-1"}} first = await router.async_pre_routing_hook( model="test-router", request_kwargs=session_kwargs(), messages=_OVERSIZED_TURNS @@ -10291,7 +10292,8 @@ class TestContextWindowEscalation: litellm_router_instance=_windowed_router(_SMALL, _BIG), complexity_router_config=_tier_config(session_affinity=True), ) - session_kwargs = lambda: {"metadata": {"session_id": "s-2", "user_api_key_hash": "k-2"}} # noqa: E731 + def session_kwargs() -> dict[str, object]: + return {"metadata": {"session_id": "s-2", "user_api_key_hash": "k-2"}} pinned = await router.async_pre_routing_hook( model="test-router", request_kwargs=session_kwargs(), messages=[{"role": "user", "content": "ok continue"}] diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 83c49afb538..f65ebd24599 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,12 +1,12 @@ { "LIT001": { - "limit": 22403 + "limit": 22402 }, "LIT002": { "limit": 26780 }, "LIT003": { - "limit": 269 + "limit": 268 }, "LIT004": { "limit": 40 @@ -27,10 +27,10 @@ "limit": 0 }, "LIT010": { - "limit": 16512 + "limit": 16511 }, "LIT011": { - "limit": 5537 + "limit": 5535 }, "LIT012": { "limit": 4495 From 7ca035f310f891eea216f3162191739f3720bcb1 Mon Sep 17 00:00:00 2001 From: Kris Xia Date: Tue, 1 Sep 2026 11:39:33 +0800 Subject: [PATCH 014/175] fix(gemini): return enabled thinking content by default --- .../gemini/vertex_and_google_ai_studio_gemini.py | 2 +- .../test_vertex_and_google_ai_studio_gemini.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d8b1e7ba17c..69fe5678de9 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -949,7 +949,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # For Gemini 3+ models, use thinkingLevel instead of thinkingBudget if model and VertexGeminiConfig._is_gemini_3_or_newer(model): if thinking_enabled: - if thinking_budget is None or thinking_budget == 0: + if thinking_budget == 0: params["includeThoughts"] = False else: params["includeThoughts"] = True diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index bd07bec900f..d2788408e09 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1185,6 +1185,18 @@ def test_vertex_ai_map_thinking_param_with_budget_tokens_0(): } +def test_vertex_ai_map_thinking_param_without_budget_tokens_for_gemini_3(): + v = VertexGeminiConfig() + result = v.map_openai_params( + non_default_params={"thinking": {"type": "enabled"}}, + optional_params={}, + model="gemini-3.5-flash", + drop_params=False, + ) + + assert result["thinkingConfig"] == {"includeThoughts": True} + + def test_vertex_ai_map_tools(): v = VertexGeminiConfig() optional_params = {} From ab1161344199539bc8dec51161fb59d6d0acf703 Mon Sep 17 00:00:00 2001 From: milan Date: Wed, 5 Aug 2026 18:01:29 +0000 Subject: [PATCH 015/175] fix(bedrock): strip client_metadata from converse additionalModelRequestFields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../bedrock/chat/converse_transformation.py | 1 + .../chat/test_converse_transformation.py | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 395d99a4caa..9e40cb5ee5b 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1324,6 +1324,7 @@ class AmazonConverseConfig(BaseConfig): ) additional_request_params.pop("parallel_tool_calls", None) + additional_request_params.pop("client_metadata", None) # Only set the topK value in for models that support it additional_request_params.update(self._handle_top_k_value(model, inference_params, drop_params)) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 63f895e1819..bd68857d664 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -979,6 +979,30 @@ def test_config_blocks_do_not_leak_into_inference_config(): assert data["serviceTier"] == {"type": "priority"} +def test_client_metadata_stripped_from_converse_request(): + """``client_metadata`` sent by codex must not reach Bedrock as a passthrough model field. + + Converse forwards ``additionalModelRequestFields`` verbatim to the model, and Anthropic + rejects the request with "client_metadata: Extra inputs are not permitted". + """ + config = AmazonConverseConfig() + + data = config._transform_request_helper( + model="anthropic.claude-opus-4-8", + system_content_blocks=[], + optional_params={ + "maxTokens": 16, + "anthropic_beta": ["computer-use-2025-01-24"], + "client_metadata": {"originator": "codex_cli_rs"}, + }, + messages=None, + ) + + fields = data.get("additionalModelRequestFields", {}) + assert "client_metadata" not in fields + assert fields["anthropic_beta"] == ["computer-use-2025-01-24"] + + def test_parallel_tool_calls_config_kept_for_sonnet_5(monkeypatch): old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost From 2063c29f5d95f8dd00eef3fd7dfcbc1df05787b8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:45:26 +0000 Subject: [PATCH 016/175] fix(anthropic): upgrade legacy thinking to adaptive on adaptive-only models for chat and Bedrock Converse Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/anthropic/chat/transformation.py | 3 + litellm/llms/anthropic/common_utils.py | 47 +++++++++++- .../messages/transformation.py | 47 +----------- .../bedrock/chat/converse_transformation.py | 3 + .../test_anthropic_chat_transformation.py | 25 ++++++ .../chat/test_converse_transformation.py | 76 +++++++++++++++++++ 6 files changed, 154 insertions(+), 47 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index e1387a9068c..319eecfac2c 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1544,6 +1544,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): optional_params.pop("thinking", None) else: optional_params["thinking"] = value + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model=model, optional_params=optional_params, custom_llm_provider=self._resolved_provider + ) elif param == "reasoning_effort": # Accept both string ("low") and dict ({"effort": "low", # "summary": "concise"}). The Responses->Chat parser keeps the diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 9871001bf66..ede93c6deb2 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -13,7 +13,12 @@ import httpx from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm -from litellm.constants import DEFAULT_MODEL_CREATED_AT_TIME +from litellm.constants import ( + DEFAULT_MODEL_CREATED_AT_TIME, + DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, +) from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_file_ids_from_messages, ) @@ -490,6 +495,46 @@ class AnthropicModelInfo(BaseLLMModelInfo): ) optional_params.pop("thinking", None) + @staticmethod + def translate_legacy_thinking_for_adaptive_model( + model: str, + optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param, as in maybe_drop_disabled_thinking + custom_llm_provider: str, + ) -> None: + """Translate legacy ``thinking.type=enabled`` to adaptive for the + adaptive-thinking models that reject it (4.7+ and the 5 families). + Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the + legacy shape natively, so it is forwarded verbatim and the caller's + ``budget_tokens`` cap keeps applying. Caller-provided + ``output_config.effort`` is never overridden. + """ + if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider): + return + if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider): + return + thinking: Final = optional_params.get("thinking") + if not isinstance(thinking, dict) or thinking.get("type") != "enabled": + return + + budget: Final = int(thinking.get("budget_tokens") or 0) + if budget >= DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET and ( + AnthropicModelInfo._supports_model_capability(model, "supports_xhigh_reasoning_effort", custom_llm_provider) + ): + effort = "xhigh" + elif budget >= DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET: + effort = "high" + elif budget >= DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET: + effort = "medium" + else: + effort = "low" + + optional_params["thinking"] = {"type": "adaptive"} + existing_output_config = optional_params.get("output_config") + if not isinstance(existing_output_config, dict): + existing_output_config = {} + existing_output_config.setdefault("effort", effort) + optional_params["output_config"] = existing_output_config + def is_effort_used( self, optional_params: dict | None, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 3d62b8b4784..988f81c9eb4 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -3,11 +3,6 @@ from typing import Any, Final import httpx -from litellm.constants import ( - DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, - DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, -) from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import verbose_logger @@ -400,46 +395,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): existing_output_config.setdefault("effort", mapped_effort) optional_params["output_config"] = existing_output_config - @staticmethod - def _translate_legacy_thinking_for_adaptive_model( - model: str, optional_params: dict, custom_llm_provider: str - ) -> None: - """Translate legacy ``thinking.type=enabled`` to adaptive for the - adaptive-thinking models that reject it (4.7+ and the 5 families). - Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the - legacy shape natively, so it is forwarded verbatim and the caller's - ``budget_tokens`` cap keeps applying. Caller-provided - ``output_config.effort`` is never overridden. - """ - from litellm.llms.anthropic.chat.transformation import AnthropicConfig - - if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider): - return - if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider): - return - thinking: Final = optional_params.get("thinking") - if not isinstance(thinking, dict) or thinking.get("type") != "enabled": - return - - budget: Final = int(thinking.get("budget_tokens") or 0) - if budget >= DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET and ( - AnthropicConfig._supports_effort_level(model, "xhigh", custom_llm_provider) - ): - effort = "xhigh" - elif budget >= DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET: - effort = "high" - elif budget >= DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET: - effort = "medium" - else: - effort = "low" - - optional_params["thinking"] = {"type": "adaptive"} - existing_output_config = optional_params.get("output_config") - if not isinstance(existing_output_config, dict): - existing_output_config = {} - existing_output_config.setdefault("effort", effort) - optional_params["output_config"] = existing_output_config - @staticmethod def _translate_adaptive_effort_for_non_adaptive_model( model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str @@ -606,7 +561,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): custom_llm_provider=self._resolved_provider, ) - self._translate_legacy_thinking_for_adaptive_model( + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( model=model, optional_params=anthropic_messages_optional_request_params, custom_llm_provider=self._resolved_provider, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 395d99a4caa..0a378dfc11c 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -934,6 +934,9 @@ class AmazonConverseConfig(BaseConfig): litellm.verbose_logger.warning(DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, model) else: optional_params["thinking"] = value + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model=model, optional_params=optional_params, custom_llm_provider="bedrock" + ) elif param == "reasoning_effort" and isinstance(value, str): self._handle_reasoning_effort_parameter( model=model, reasoning_effort=value, optional_params=optional_params diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 25e2c3cda80..4f30e7d10f0 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3127,6 +3127,31 @@ def test_reasoning_effort_accepts_dict_shape_for_non_adaptive_model( ) +@pytest.mark.parametrize( + "model,budget_tokens,expected", + [ + ("claude-opus-4-8", 4096, ({"type": "adaptive"}, {"effort": "high"})), + ("claude-opus-4-7", 24000, ({"type": "adaptive"}, {"effort": "xhigh"})), + ("claude-opus-4-6", 4096, ({"type": "enabled", "budget_tokens": 4096}, None)), + ("claude-sonnet-4-5-20250929", 4096, ({"type": "enabled", "budget_tokens": 4096}, None)), + ], +) +def test_legacy_thinking_translated_to_adaptive_on_adaptive_only_models(model, budget_tokens, expected): + """Adaptive-only models reject thinking={type: enabled} with a 400, so the + legacy shape must be upgraded to adaptive + output_config.effort on + /chat/completions too, while models that accept it keep the caller's budget.""" + config = AnthropicConfig() + + result = config.map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": budget_tokens}, "max_tokens": 64000}, + optional_params={}, + model=model, + drop_params=False, + ) + + assert (result["thinking"], result.get("output_config")) == expected + + @pytest.mark.parametrize( "bad_value", [ diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 63f895e1819..b729c9366cc 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -6269,6 +6269,82 @@ def test_adaptive_thinking_passes_through_on_46_plus_converse(model): assert optional_params.get("thinking") == {"type": "adaptive"} +@pytest.mark.parametrize( + "model,budget_tokens,expected_effort", + [ + ("anthropic.claude-opus-4-8", 4096, "high"), + ("us.anthropic.claude-opus-4-8", 2000, "low"), + ("global.anthropic.claude-opus-4-8", 12000, "xhigh"), + ("us.anthropic.claude-opus-4-7", 3000, "medium"), + ("anthropic.claude-fable-5", 4096, "high"), + ], +) +def test_legacy_thinking_translated_to_adaptive_on_adaptive_only_converse(model, budget_tokens, expected_effort): + """Adaptive-only models (4.7+, 5 families) reject thinking={type: enabled} + with a 400 on Bedrock Converse, so the legacy shape from callers like Claude + Code must be upgraded to thinking={type: adaptive} + output_config.effort + derived from budget_tokens, matching the /v1/messages passthrough.""" + config = AmazonConverseConfig() + + optional_params = config.map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": budget_tokens}, "max_tokens": 64000}, + optional_params={}, + model=model, + drop_params=False, + ) + request = config.transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request["additionalModelRequestFields"]["thinking"] == {"type": "adaptive"} + assert request["additionalModelRequestFields"]["output_config"] == {"effort": expected_effort} + + +def test_legacy_thinking_translation_keeps_caller_output_config_effort_converse(): + config = AmazonConverseConfig() + + optional_params = config.map_openai_params( + non_default_params={ + "output_config": {"effort": "low"}, + "thinking": {"type": "enabled", "budget_tokens": 12000}, + "max_tokens": 64000, + }, + optional_params={}, + model="anthropic.claude-opus-4-8", + drop_params=False, + ) + + assert optional_params["thinking"] == {"type": "adaptive"} + assert optional_params["output_config"] == {"effort": "low"} + + +@pytest.mark.parametrize( + "model", + [ + "us.anthropic.claude-opus-4-6", + "anthropic.claude-3-5-sonnet-20241022-v2:0", + ], +) +def test_legacy_thinking_forwarded_verbatim_when_model_accepts_it_converse(model): + """The 4.6 family and pre-adaptive models accept thinking={type: enabled} + natively, so the caller's budget_tokens cap must keep applying.""" + config = AmazonConverseConfig() + + optional_params = config.map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 4096}, "max_tokens": 8192}, + optional_params={}, + model=model, + drop_params=False, + ) + + assert optional_params["thinking"] == {"type": "enabled", "budget_tokens": 4096} + assert "output_config" not in optional_params + + def test_adaptive_thinking_dropped_when_max_tokens_too_small_converse(): """When max_tokens can't fit even the minimum thinking budget, the raw adaptive block must be dropped entirely rather than translated, so the From b8dd27a77fdaaa390761a99bf27737a255c37e3a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:00:46 +0000 Subject: [PATCH 017/175] style(anthropic): keep mutable-ok annotation within ruff format width Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/anthropic/common_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index ede93c6deb2..f2fcbc6c232 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -498,7 +498,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): @staticmethod def translate_legacy_thinking_for_adaptive_model( model: str, - optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param, as in maybe_drop_disabled_thinking + optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param like the sibling helpers custom_llm_provider: str, ) -> None: """Translate legacy ``thinking.type=enabled`` to adaptive for the From ed4343a02645e19590657ae257e6ae43b95f8e48 Mon Sep 17 00:00:00 2001 From: milan Date: Tue, 1 Sep 2026 19:04:32 +0000 Subject: [PATCH 018/175] fix(bedrock): scope client_metadata drop to anthropic converse models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../bedrock/chat/converse_transformation.py | 8 ++++++- .../chat/test_converse_transformation.py | 24 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 9e40cb5ee5b..8d1d905a129 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1324,7 +1324,13 @@ class AmazonConverseConfig(BaseConfig): ) additional_request_params.pop("parallel_tool_calls", None) - additional_request_params.pop("client_metadata", None) + + if base_model.startswith("anthropic") and additional_request_params.pop("client_metadata", None) is not None: + litellm.verbose_logger.debug( + "Bedrock Converse: dropping `client_metadata` for model=%s, Anthropic rejects it with " + "'client_metadata: Extra inputs are not permitted'", + model, + ) # Only set the topK value in for models that support it additional_request_params.update(self._handle_top_k_value(model, inference_params, drop_params)) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index bd68857d664..3600e366b5a 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -979,8 +979,9 @@ def test_config_blocks_do_not_leak_into_inference_config(): assert data["serviceTier"] == {"type": "priority"} -def test_client_metadata_stripped_from_converse_request(): - """``client_metadata`` sent by codex must not reach Bedrock as a passthrough model field. +@pytest.mark.parametrize("model", ["anthropic.claude-opus-4-8", "us.anthropic.claude-opus-4-8"]) +def test_client_metadata_stripped_for_anthropic_converse_request(model): + """``client_metadata`` sent by codex must not reach Anthropic as a passthrough model field. Converse forwards ``additionalModelRequestFields`` verbatim to the model, and Anthropic rejects the request with "client_metadata: Extra inputs are not permitted". @@ -988,7 +989,7 @@ def test_client_metadata_stripped_from_converse_request(): config = AmazonConverseConfig() data = config._transform_request_helper( - model="anthropic.claude-opus-4-8", + model=model, system_content_blocks=[], optional_params={ "maxTokens": 16, @@ -1003,6 +1004,23 @@ def test_client_metadata_stripped_from_converse_request(): assert fields["anthropic_beta"] == ["computer-use-2025-01-24"] +def test_client_metadata_kept_for_non_anthropic_converse_request(): + """Only Anthropic is known to reject ``client_metadata``, so other families keep the passthrough.""" + config = AmazonConverseConfig() + + data = config._transform_request_helper( + model="amazon.nova-pro-v1:0", + system_content_blocks=[], + optional_params={ + "maxTokens": 16, + "client_metadata": {"originator": "codex_cli_rs"}, + }, + messages=None, + ) + + assert data["additionalModelRequestFields"]["client_metadata"] == {"originator": "codex_cli_rs"} + + def test_parallel_tool_calls_config_kept_for_sonnet_5(monkeypatch): old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost From b96121efb173965405aa521ffbbba026ce73d3a4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:10:22 +0000 Subject: [PATCH 019/175] refactor(anthropic): build adaptive output_config in one shot in legacy thinking helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/anthropic/common_utils.py | 37 +++++++++++++++----------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index f2fcbc6c232..17fa8022388 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -516,24 +516,29 @@ class AnthropicModelInfo(BaseLLMModelInfo): if not isinstance(thinking, dict) or thinking.get("type") != "enabled": return - budget: Final = int(thinking.get("budget_tokens") or 0) - if budget >= DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET and ( + effort: Final = AnthropicModelInfo._legacy_budget_to_effort( + model=model, + budget_tokens=int(thinking.get("budget_tokens") or 0), + custom_llm_provider=custom_llm_provider, + ) + existing_output_config: Final = optional_params.get("output_config") + optional_params["thinking"] = {"type": "adaptive"} + optional_params["output_config"] = { + "effort": effort, + **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})), + } + + @staticmethod + def _legacy_budget_to_effort(model: str, budget_tokens: int, custom_llm_provider: str) -> str: + if budget_tokens >= DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET and ( AnthropicModelInfo._supports_model_capability(model, "supports_xhigh_reasoning_effort", custom_llm_provider) ): - effort = "xhigh" - elif budget >= DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET: - effort = "high" - elif budget >= DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET: - effort = "medium" - else: - effort = "low" - - optional_params["thinking"] = {"type": "adaptive"} - existing_output_config = optional_params.get("output_config") - if not isinstance(existing_output_config, dict): - existing_output_config = {} - existing_output_config.setdefault("effort", effort) - optional_params["output_config"] = existing_output_config + return "xhigh" + if budget_tokens >= DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET: + return "high" + if budget_tokens >= DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET: + return "medium" + return "low" def is_effort_used( self, From bba951c5ebe20871caab9848c474585c91fa3535 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 1 Sep 2026 19:26:19 +0000 Subject: [PATCH 020/175] fix(bedrock): drop client_metadata for ARNs that hide the model family Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../bedrock/chat/converse_transformation.py | 4 +- litellm/llms/bedrock/common_utils.py | 9 ++++ .../chat/test_converse_transformation.py | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 8d1d905a129..99d45bfc94b 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -86,6 +86,7 @@ from litellm.utils import ( from ..common_utils import ( BedrockError, BedrockModelInfo, + bedrock_arn_hides_model_family, bedrock_converse_supports_parallel_tool_use_config, get_anthropic_beta_from_headers, get_bedrock_tool_name, @@ -1325,7 +1326,8 @@ class AmazonConverseConfig(BaseConfig): additional_request_params.pop("parallel_tool_calls", None) - if base_model.startswith("anthropic") and additional_request_params.pop("client_metadata", None) is not None: + drops_client_metadata: Final = base_model.startswith("anthropic") or bedrock_arn_hides_model_family(model) + if drops_client_metadata and additional_request_params.pop("client_metadata", None) is not None: litellm.verbose_logger.debug( "Bedrock Converse: dropping `client_metadata` for model=%s, Anthropic rejects it with " "'client_metadata: Extra inputs are not permitted'", diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 9cbceb4880c..3b82d98ceee 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -720,6 +720,15 @@ def get_bedrock_base_model(model: str) -> str: return model +def bedrock_arn_hides_model_family(model: str) -> bool: + """ + True for an ARN-addressed model whose base name carries no ``provider.model`` + id, such as an application inference profile or a provisioned throughput ARN. + Callers that gate behavior on the model family cannot resolve one here. + """ + return "arn:" in model.lower() and "." not in get_bedrock_base_model(model) + + def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool: return any( (litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 3600e366b5a..423e3a5a7c3 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -1021,6 +1021,49 @@ def test_client_metadata_kept_for_non_anthropic_converse_request(): assert data["additionalModelRequestFields"]["client_metadata"] == {"originator": "codex_cli_rs"} +@pytest.mark.parametrize( + "model", + [ + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abcdef123456", + "arn:aws:bedrock:us-east-1:123456789012:provisioned-model/abcdef123456", + ], +) +def test_client_metadata_stripped_for_arn_models_converse(model): + """An ARN hides which family serves the request, and pointing one at Claude is how + teams route codex traffic, so the field has to go there too or the 400 comes back.""" + config = AmazonConverseConfig() + + data = config._transform_request_helper( + model=model, + system_content_blocks=[], + optional_params={ + "maxTokens": 16, + "client_metadata": {"originator": "codex_cli_rs"}, + }, + messages=None, + ) + + assert "client_metadata" not in data.get("additionalModelRequestFields", {}) + + +def test_client_metadata_kept_for_arn_naming_a_non_anthropic_family(): + """An inference profile ARN that still spells out the family is resolvable, so a + non-Anthropic one keeps its passthrough.""" + config = AmazonConverseConfig() + + data = config._transform_request_helper( + model="arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.amazon.nova-pro-v1:0", + system_content_blocks=[], + optional_params={ + "maxTokens": 16, + "client_metadata": {"originator": "codex_cli_rs"}, + }, + messages=None, + ) + + assert data["additionalModelRequestFields"]["client_metadata"] == {"originator": "codex_cli_rs"} + + def test_parallel_tool_calls_config_kept_for_sonnet_5(monkeypatch): old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost From 24ee419c85f6758369e6582250a50490ce1d6819 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:28:04 +0000 Subject: [PATCH 021/175] fix(models): registry audit 2026-09-01 for openai realtime, mistral aliases, voyage, xai, fireworks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 133 ++++++++++++++---- model_prices_and_context_window.json | 133 ++++++++++++++---- 2 files changed, 212 insertions(+), 54 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 27ff525c15e..08add22c998 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -30606,17 +30606,18 @@ }, "gpt-realtime-2": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, - "output_cost_per_token": 1.6e-05, + "output_cost_per_token": 2.4e-05, "supported_endpoints": [ "/v1/realtime" ], @@ -30680,8 +30681,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -30713,7 +30714,7 @@ "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", - "max_input_tokens": 128000, + "max_input_tokens": 32000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", @@ -33477,19 +33478,21 @@ "source": "https://mistral.ai/pricing#api-pricing" }, "mistral/magistral-medium-latest": { - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-2506": { "deprecation_date": "2025-11-30", @@ -33508,19 +33511,21 @@ "supports_tool_choice": true }, "mistral/magistral-small-latest": { - "input_cost_per_token": 5e-07, + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", + "output_cost_per_token": 6e-07, + "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-1-2-2509": { "deprecation_date": "2026-07-31", @@ -33652,16 +33657,21 @@ "supports_vision": true }, "mistral/mistral-medium": { - "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 8.1e-06, + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/mistral-medium-2312": { "deprecation_date": "2025-06-16", @@ -45539,6 +45549,26 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, + "voyage/rerank-3": { + "input_cost_per_token": 5e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, + "voyage/rerank-3-lite": { + "input_cost_per_token": 2e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, "voyage/voyage-2": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", @@ -46790,6 +46820,27 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-build-latest": { + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "xai", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://docs.x.ai/developers/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "xai/grok-4.6": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, @@ -57110,6 +57161,34 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/glm-5p3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/accounts/fireworks/models/inkling": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 4.05e-06, + "source": "https://fireworks.ai/models/fireworks/inkling", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-8b": { "input_cost_per_token": 1e-07, "output_cost_per_token": 0.0, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 27ff525c15e..08add22c998 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -30606,17 +30606,18 @@ }, "gpt-realtime-2": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, - "output_cost_per_token": 1.6e-05, + "output_cost_per_token": 2.4e-05, "supported_endpoints": [ "/v1/realtime" ], @@ -30680,8 +30681,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -30713,7 +30714,7 @@ "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", - "max_input_tokens": 128000, + "max_input_tokens": 32000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", @@ -33477,19 +33478,21 @@ "source": "https://mistral.ai/pricing#api-pricing" }, "mistral/magistral-medium-latest": { - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-2506": { "deprecation_date": "2025-11-30", @@ -33508,19 +33511,21 @@ "supports_tool_choice": true }, "mistral/magistral-small-latest": { - "input_cost_per_token": 5e-07, + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", + "output_cost_per_token": 6e-07, + "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-1-2-2509": { "deprecation_date": "2026-07-31", @@ -33652,16 +33657,21 @@ "supports_vision": true }, "mistral/mistral-medium": { - "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 8.1e-06, + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/mistral-medium-2312": { "deprecation_date": "2025-06-16", @@ -45539,6 +45549,26 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, + "voyage/rerank-3": { + "input_cost_per_token": 5e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, + "voyage/rerank-3-lite": { + "input_cost_per_token": 2e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, "voyage/voyage-2": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", @@ -46790,6 +46820,27 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-build-latest": { + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "xai", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://docs.x.ai/developers/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "xai/grok-4.6": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, @@ -57110,6 +57161,34 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/glm-5p3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/accounts/fireworks/models/inkling": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 4.05e-06, + "source": "https://fireworks.ai/models/fireworks/inkling", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-8b": { "input_cost_per_token": 1e-07, "output_cost_per_token": 0.0, From cf4738c3b736edb62f111ab26d899da6a7f9f284 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:32:04 -0700 Subject: [PATCH 022/175] style: format s3 vectors transformation and rag endpoints --- litellm/llms/s3_vectors/vector_stores/transformation.py | 4 +++- litellm/proxy/rag_endpoints/endpoints.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/llms/s3_vectors/vector_stores/transformation.py b/litellm/llms/s3_vectors/vector_stores/transformation.py index 3c1b0025d08..733358381fe 100644 --- a/litellm/llms/s3_vectors/vector_stores/transformation.py +++ b/litellm/llms/s3_vectors/vector_stores/transformation.py @@ -68,7 +68,9 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): """Return the router iff it serves ``embedding_model`` as a deployment.""" if router is None: return None - model_list: Final = [dict(m) for m in (router.get_model_list() or ())] # mutable-ok: resolve_embedding_router requires list[dict] + model_list: Final = [ + dict(m) for m in (router.get_model_list() or ()) + ] # mutable-ok: resolve_embedding_router requires list[dict] return resolve_embedding_router(embedding_model=embedding_model, llm_router=router, llm_model_list=model_list) def transform_search_vector_store_request( diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index 31998d2040e..5c392c30018 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -717,7 +717,10 @@ async def rag_query( vector_store_id=retrieval_config["vector_store_id"], user_api_key_dict=user_api_key_dict, ) - merged_retrieval_config: Final = {**store_data, **retrieval_config} # mutable-ok: litellm.aquery requires a plain dict payload + merged_retrieval_config: Final = { + **store_data, + **retrieval_config, + } # mutable-ok: litellm.aquery requires a plain dict payload # Add litellm data request_data: dict[str, object] = {} From d3dab8e294b06badb2d34f31ce6aade9380a49ea Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:46:04 -0700 Subject: [PATCH 023/175] fix(rerank): map provider errors with the resolved provider on sync and async paths --- litellm/rerank_api/main.py | 22 ++++++-- tests/test_litellm/rerank_api/test_main.py | 61 ++++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index c8f7842aebf..597d1cfb863 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -43,10 +43,17 @@ async def arerank( """ Async: Reranks a list of documents based on their relevance to the query """ + _custom_llm_provider: str | None = None # rebind-ok: set by the get_llm_provider unpack; read in the except try: loop: Final = asyncio.get_event_loop() kwargs["arerank"] = True + _, _custom_llm_provider, _, _ = litellm.get_llm_provider( # rebind-ok: see pre-declaration above + model=model, + custom_llm_provider=custom_llm_provider, + api_base=kwargs.get("api_base", None), + ) + func: Final = partial( rerank, model, @@ -70,7 +77,11 @@ async def arerank( response = init_response return response except Exception as e: - raise e + raise exception_type( + model=model, + custom_llm_provider=_custom_llm_provider or custom_llm_provider, + original_exception=e, + ) @client @@ -115,6 +126,7 @@ def rerank( model_info: Final = kwargs.get("model_info", None) user: Final = kwargs.get("user", None) client: Final = kwargs.get("client", None) + _custom_llm_provider: str | None = None # rebind-ok: set by the get_llm_provider unpack; read in the except try: _is_async: Final = kwargs.pop("arerank", False) is True optional_params: Final = GenericLiteLLMParams(**kwargs) @@ -127,7 +139,7 @@ def rerank( ( model, - _custom_llm_provider, + _custom_llm_provider, # rebind-ok: see pre-declaration above dynamic_api_key, dynamic_api_base, ) = litellm.get_llm_provider( @@ -538,4 +550,8 @@ def rerank( return response except Exception as e: verbose_logger.error("Error in rerank: %s", e) - raise exception_type(model=model, custom_llm_provider=custom_llm_provider, original_exception=e) + raise exception_type( + model=model, + custom_llm_provider=_custom_llm_provider or custom_llm_provider, + original_exception=e, + ) diff --git a/tests/test_litellm/rerank_api/test_main.py b/tests/test_litellm/rerank_api/test_main.py index 587be59c550..62149c742d6 100644 --- a/tests/test_litellm/rerank_api/test_main.py +++ b/tests/test_litellm/rerank_api/test_main.py @@ -111,6 +111,67 @@ def test_together_rerank_honors_api_base(respx_mock: respx.MockRouter): assert mock_route.calls[0].request.headers["authorization"] == "Bearer fake-together-key" +DASHSCOPE_404_BODY = { + "error": { + "message": "The model `does-not-exist` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": None, + "code": "model_not_found", + }, + "request_id": "mock-request-id", +} + + +def test_rerank_error_names_provider_and_keeps_body(respx_mock: respx.MockRouter, monkeypatch): + """Regression for the rerank error path mapping with the unresolved provider param: + a provider 404 surfaced as 'None - ' instead of naming the provider and its error body.""" + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False) + + mock_route = respx_mock.post("https://dashscope.example/v1/reranks") + mock_route.return_value = httpx.Response(404, json=DASHSCOPE_404_BODY) + + with pytest.raises(litellm.NotFoundError) as exc_info: + litellm.rerank( + model="dashscope/does-not-exist", + query=MARKER_QUERY, + documents=[MARKER_DOC], + api_key="fake-dashscope-key", + api_base="https://dashscope.example/v1", + ) + + assert mock_route.called + assert "DashscopeException" in str(exc_info.value) + assert "does not exist or you do not have access to it" in str(exc_info.value) + assert "None - " not in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_arerank_error_is_mapped_to_litellm_exception(respx_mock: respx.MockRouter, monkeypatch): + """Regression for arerank's bare re-raise: provider errors escaped as raw + provider exception classes instead of the mapped litellm exception contract.""" + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + + mock_route = respx_mock.post("https://dashscope.example/v1/reranks") + mock_route.return_value = httpx.Response(404, json=DASHSCOPE_404_BODY) + + with pytest.raises(litellm.NotFoundError) as exc_info: + await litellm.arerank( + model="dashscope/does-not-exist", + query=MARKER_QUERY, + documents=[MARKER_DOC], + api_key="fake-dashscope-key", + api_base="https://dashscope.example/v1", + ) + + assert mock_route.called + assert "DashscopeException" in str(exc_info.value) + assert "does not exist or you do not have access to it" in str(exc_info.value) + assert "None - " not in str(exc_info.value) + + @pytest.mark.asyncio async def test_together_rerank_async_honors_env_api_base(respx_mock: respx.MockRouter, monkeypatch): """Regression: TOGETHER_AI_API_BASE was honored by chat but ignored by rerank.""" From 3914de24eff6c0f3deda46ec2f2217cb0305d916 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:52:45 -0700 Subject: [PATCH 024/175] fix(router): route model-less sync vector store calls to the SDK _generic_api_call_with_fallbacks requires a model, so sync vector_store_search and vector_store_create raised a TypeError whenever the call carried no model. Model-less calls now go directly to the SDK function, with the router injected for search, matching the async wrapper's behavior --- litellm/router.py | 19 +++++++---- tests/test_litellm/test_router.py | 55 +++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d8769525815..96d5b1e1488 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6299,8 +6299,6 @@ class Router: "responses", "generate_content", "generate_content_stream", - "vector_store_search", - "vector_store_create", "ocr", "search", "video_generation", @@ -6324,6 +6322,8 @@ class Router: return sync_wrapper if call_type in ( + "vector_store_search", + "vector_store_create", "vector_store_retrieve", "vector_store_list", "vector_store_update", @@ -6335,11 +6335,16 @@ class Router: client: object | None = None, **kwargs, ): - if custom_llm_provider and "custom_llm_provider" not in kwargs: - kwargs["custom_llm_provider"] = custom_llm_provider - if kwargs.get("model"): - return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs) - return original_function(**kwargs) + provider_kwargs: Final = ( + MappingProxyType({**kwargs, "custom_llm_provider": custom_llm_provider}) + if custom_llm_provider and "custom_llm_provider" not in kwargs + else MappingProxyType(kwargs) + ) + if provider_kwargs.get("model"): + return self._generic_api_call_with_fallbacks(original_function=original_function, **provider_kwargs) + if call_type == "vector_store_search": + return original_function(**MappingProxyType({**provider_kwargs, "router": self})) + return original_function(**provider_kwargs) return vector_store_sync_wrapper diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c222fab79f0..3abfe8ce522 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7567,6 +7567,61 @@ async def test_avector_store_create_does_not_inject_router(): assert "router" not in mock_acreate.await_args.kwargs +def test_vector_store_search_injects_router(): + """ + Sync parity for the router injection: router.vector_store_search must pass + the router down to the SDK search call so provider transforms can resolve + router-managed embedding models, same as avector_store_search. + """ + from litellm.types.vector_stores import VectorStoreSearchResponse + + expected_response = VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + mock_search = MagicMock(return_value=expected_response) + # Router.__init__ binds search via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.search", new=mock_search): # test-quality-ok: the SDK call is the only place the injected router kwarg is observable + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + search_response = router.vector_store_search( + vector_store_id="v", query="q", custom_llm_provider="s3_vectors" + ) + + assert search_response is expected_response + mock_search.assert_called_once() + assert mock_search.call_args.kwargs["router"] is router + assert mock_search.call_args.kwargs["custom_llm_provider"] == "s3_vectors" + + +def test_vector_store_create_does_not_inject_router(): + """The sync create path must keep calling the SDK without a router kwarg.""" + expected_response = {"id": "vs_1", "object": "vector_store"} + mock_create = MagicMock(return_value=expected_response) + # Router.__init__ binds create via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.create", new=mock_create): # test-quality-ok: the SDK call is the only place a leaked router kwarg would surface + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + create_response = router.vector_store_create(custom_llm_provider="openai") + + assert create_response is expected_response + mock_create.assert_called_once() + assert "router" not in mock_create.call_args.kwargs + + class TestPreRoutingStrategyRegistryLifecycle: """ Regression tests: a deployment leaving the model_list must release the From babe7816ada8d622be993b542fc2512037d2466f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:52:45 -0700 Subject: [PATCH 025/175] fix(rag): store-wins merge, single lookup, allowlisted search params rag_query reuses the store resolved during authorization instead of a second registry lookup, merges registry data store-wins so callers cannot override a managed store's provider or credentials, and logs ids instead of the merged config, which can carry resolved credentials. aquery forwards only allowlisted retrieval_config keys to vector store search, keeping caller-supplied connection overrides like api_base and api_key away from the search call --- litellm/proxy/rag_endpoints/endpoints.py | 53 ++++++++---- .../proxy/vector_store_endpoints/endpoints.py | 86 ++++++++++--------- .../management_endpoints.py | 4 +- litellm/rag/main.py | 25 ++++-- .../proxy/rag_endpoints/test_rag_endpoints.py | 26 +++--- tests/test_litellm/rag/test_main.py | 39 +++++++++ 6 files changed, 149 insertions(+), 84 deletions(-) diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index 5c392c30018..d2c7d6f93ee 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -9,6 +9,7 @@ Provides: import base64 import json from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import orjson @@ -19,6 +20,9 @@ from starlette.datastructures import UploadFile import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + LiteLLM_ManagedVectorStore, +) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.proxy._types import * from litellm.proxy.auth.auth_utils import is_request_body_safe @@ -37,7 +41,7 @@ from litellm.proxy.rag_endpoints.upload_security import ( validate_upload, ) from litellm.proxy.vector_store_endpoints.endpoints import ( - _update_request_data_with_litellm_managed_vector_store_registry, # pyright: ignore[reportPrivateUsage] # shared registry-merge helper used by the direct search endpoint + build_request_data_from_managed_vector_store, ) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, @@ -123,12 +127,21 @@ def _collect_vector_store_ids_from_payload(payload: object) -> set[str]: async def _authorize_nested_vector_store_ids( payload: object, user_api_key_dict: UserAPIKeyAuth, -) -> None: - for vector_store_id in sorted(_collect_vector_store_ids_from_payload(payload)): - await assert_user_can_access_vector_store_id( - vector_store_id=vector_store_id, - user_api_key_dict=user_api_key_dict, - ) +) -> Mapping[str, LiteLLM_ManagedVectorStore]: + """Authorize every nested vector store id and return the managed stores it resolved.""" + return MappingProxyType( + { + vector_store_id: store + for vector_store_id in sorted(_collect_vector_store_ids_from_payload(payload)) + if ( + store := await assert_user_can_access_vector_store_id( + vector_store_id=vector_store_id, + user_api_key_dict=user_api_key_dict, + ) + ) + is not None + } + ) def _build_file_metadata_entry( @@ -703,23 +716,24 @@ async def rag_query( status_code=400, detail={"error": "retrieval_config must contain 'vector_store_id'"}, ) - await _authorize_nested_vector_store_ids( + resolved_stores: Final = await _authorize_nested_vector_store_ids( payload=retrieval_config, user_api_key_dict=user_api_key_dict, ) # Merge litellm-managed vector store params (provider, region, embedding - # model, credentials, ...) from the registry — same source the direct - # /vector_stores/{id}/search endpoint uses. User-supplied - # retrieval_config keys win on conflict. - store_data: Final = await _update_request_data_with_litellm_managed_vector_store_registry( - data={}, # mutable-ok: the helper mutates and returns the seed dict - vector_store_id=retrieval_config["vector_store_id"], - user_api_key_dict=user_api_key_dict, + # model, credentials, ...) from the registry: the same source the direct + # /vector_stores/{id}/search endpoint uses. Store-managed keys win on + # conflict so callers cannot override the store's provider or credentials. + managed_store: Final = resolved_stores.get(retrieval_config["vector_store_id"]) + store_data: Final = ( + await build_request_data_from_managed_vector_store(managed_store) + if managed_store is not None + else MappingProxyType({}) ) merged_retrieval_config: Final = { - **store_data, **retrieval_config, + **store_data, } # mutable-ok: litellm.aquery requires a plain dict payload # Add litellm data @@ -733,7 +747,12 @@ async def rag_query( proxy_config=proxy_config, ) - verbose_proxy_logger.debug("RAG Query - model: %s, retrieval_config: %s", model, merged_retrieval_config) + verbose_proxy_logger.debug( + "RAG Query - model: %s, vector_store_id: %s, custom_llm_provider: %s", + model, + retrieval_config["vector_store_id"], + merged_retrieval_config.get("custom_llm_provider"), + ) # Call query response: Final = await litellm.aquery( diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index a59d7a277cc..3fc6749f18a 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import ( Annotated, Any, # noqa: TID251 # jsonify_object in proxy/utils.py is annotated with a bare dict @@ -32,6 +34,41 @@ router: Final = APIRouter() ######################################################## +async def build_request_data_from_managed_vector_store( + vector_store: LiteLLM_ManagedVectorStore, +) -> Mapping[str, object]: + """ + Build request params (provider, credential ref, litellm_params) from an + already-resolved managed vector store. + + ``litellm_embedding_config`` is resolved here, at request-handling time, + instead of at row-creation time: the resolved api_key/api_base/api_version + lives only in the returned per-request mapping and is never persisted back + to the registry cache. Legacy rows that already carry a resolved + (cleartext) config skip the lookup and pass through unchanged. + """ + top_level: Final = MappingProxyType( + { + key: vector_store.get(key) + for key in ("custom_llm_provider", "litellm_credential_name") + if key in vector_store + } + ) + litellm_params: Final = vector_store.get("litellm_params") or MappingProxyType({}) + embedding_model: Final = litellm_params.get("litellm_embedding_model") + if not embedding_model or litellm_params.get("litellm_embedding_config"): + return MappingProxyType({**top_level, **litellm_params}) + + from litellm.proxy.proxy_server import prisma_client + + resolved_config: Final = await _resolve_embedding_config( + embedding_model=embedding_model, prisma_client=prisma_client + ) + if not resolved_config: + return MappingProxyType({**top_level, **litellm_params}) + return MappingProxyType({**top_level, **litellm_params, "litellm_embedding_config": resolved_config}) + + async def _update_request_data_with_litellm_managed_vector_store_registry( data: dict, vector_store_id: str, @@ -51,47 +88,14 @@ async def _update_request_data_with_litellm_managed_vector_store_registry( vector_store_to_run: Final[LiteLLM_ManagedVectorStore | None] = await get_litellm_managed_vector_store( vector_store_id=vector_store_id ) - if vector_store_to_run is not None: - if user_api_key_dict is not None: - await assert_user_can_access_vector_store( - vector_store=vector_store_to_run, - user_api_key_dict=user_api_key_dict, - ) - - if "custom_llm_provider" in vector_store_to_run: - data["custom_llm_provider"] = vector_store_to_run.get("custom_llm_provider") - - if "litellm_credential_name" in vector_store_to_run: - data["litellm_credential_name"] = vector_store_to_run.get("litellm_credential_name") - - if "litellm_params" in vector_store_to_run: - litellm_params = vector_store_to_run.get("litellm_params", {}) or {} - # Resolve ``litellm_embedding_config`` here, at request-handling - # time, instead of at row-creation time. The resolved - # ``api_key`` / ``api_base`` / ``api_version`` lives only in - # this per-request ``data`` dict and is never persisted. - # Legacy rows that already carry a resolved (cleartext) - # ``litellm_embedding_config`` skip the lookup and pass through - # unchanged so the embed call keeps working. - embedding_model: Final = litellm_params.get("litellm_embedding_model") - if embedding_model and not litellm_params.get("litellm_embedding_config"): - from litellm.proxy.proxy_server import prisma_client - - resolved_config: Final = await _resolve_embedding_config( - embedding_model=embedding_model, prisma_client=prisma_client - ) - if resolved_config: - # Build a fresh dict via spread instead of mutating - # ``litellm_params`` in place — the registry hands back - # a reference to its cached object, so an in-place - # update would persist the resolved cleartext into the - # in-memory cache for the lifetime of the process. - litellm_params = { - **litellm_params, - "litellm_embedding_config": resolved_config, - } - data.update(litellm_params) - return data + if vector_store_to_run is None: + return data + if user_api_key_dict is not None: + await assert_user_can_access_vector_store( + vector_store=vector_store_to_run, + user_api_key_dict=user_api_key_dict, + ) + return {**data, **(await build_request_data_from_managed_vector_store(vector_store_to_run))} @router.post( diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 183a03cc13c..244798ba05e 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -470,7 +470,7 @@ async def create_vector_store_in_db( # exposed every env-stored embedding-model credential on the # ``/vector_store/{new,info,update,list}`` responses. Keep the user's # raw ``litellm_embedding_model`` reference; resolution now happens in - # ``_update_request_data_with_litellm_managed_vector_store_registry`` + # ``build_request_data_from_managed_vector_store`` # at request-handling time so the cleartext config exists only in # per-request memory and never reaches the database. if litellm_params: @@ -864,7 +864,7 @@ async def update_vector_store( # embedding-config auto-resolve previously persisted cleartext # credentials into the row; resolution now happens at request- # handling time in - # ``_update_request_data_with_litellm_managed_vector_store_registry`` + # ``build_request_data_from_managed_vector_store`` # so this row only ever stores the user-supplied # ``litellm_embedding_model`` reference. if "litellm_params" in update_data: diff --git a/litellm/rag/main.py b/litellm/rag/main.py index bd6788b3a1b..94bfc305a6a 100644 --- a/litellm/rag/main.py +++ b/litellm/rag/main.py @@ -51,12 +51,19 @@ INGESTION_REGISTRY: Final[dict[str, type[BaseRAGIngestion]]] = { "vertex_ai": VertexAIRAGIngestion, } -# retrieval_config keys consumed by the query pipeline itself; everything else is -# forwarded to vector_stores.asearch as provider-specific params (e.g. -# aws_region_name, embedding_model, vector_bucket_name for S3 Vectors). -# `filters`/`retrieval_filter` are reserved for the explicit filter param. -_CONSUMED_RETRIEVAL_CONFIG_KEYS: Final = frozenset( - {"vector_store_id", "custom_llm_provider", "top_k", "filters", "retrieval_filter"} +# Only these retrieval_config keys are forwarded to vector_stores.asearch as +# provider-specific params. The explicit allowlist keeps caller-controlled +# connection overrides (api_base, api_key, ...) away from the search call, +# where they could redirect store credentials to an attacker-chosen host. +_FORWARDABLE_RETRIEVAL_CONFIG_KEYS: Final = frozenset( + { + "aws_region_name", + "vector_bucket_name", + "embedding_model", + "litellm_embedding_model", + "litellm_embedding_config", + "litellm_credential_name", + } ) @@ -233,10 +240,10 @@ async def _execute_query_pipeline( raise ValueError("No query found in messages for RAG query") # 2. Search vector store - # Forward provider-specific retrieval_config extras (region, embedding model, - # bucket, credentials refs, ...) to the search call; kwargs win on conflict. + # Forward allowlisted provider retrieval_config extras (region, embedding + # model, bucket, credential refs) to the search call; kwargs win on conflict. provider_search_params: Final = MappingProxyType( - {k: v for k, v in retrieval_config.items() if k not in _CONSUMED_RETRIEVAL_CONFIG_KEYS} + {k: v for k, v in retrieval_config.items() if k in _FORWARDABLE_RETRIEVAL_CONFIG_KEYS} ) forwarded_search_params: Final = MappingProxyType({**provider_search_params, **kwargs}) with _suppressed_sub_call_billing(): diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 5561ee1e6ae..342a4535b21 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -357,12 +357,9 @@ def test_rag_query_merges_managed_store_params(client_internal_user): "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", new_callable=AsyncMock, return_value=mock_response, - ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and stubs the access assert covered by auth tests - "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", - new=AsyncMock(), - ), patch( # test-quality-ok: stubs the direct-endpoint access assert covered by auth tests - "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", - new=AsyncMock(), + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and grants access so real store resolution runs + "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", + new=AsyncMock(return_value=True), ): response = client_internal_user.post( "/v1/rag/query", @@ -383,8 +380,8 @@ def test_rag_query_merges_managed_store_params(client_internal_user): assert forwarded_config["vector_bucket_name"] == "bkt" -def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): - """User-supplied retrieval_config keys must win over registry values.""" +def test_rag_query_store_params_win_over_user_retrieval_config(client_internal_user): + """Registry values must win over user-supplied retrieval_config keys so callers cannot override store credentials.""" import litellm from litellm.types.utils import ModelResponse @@ -406,12 +403,9 @@ def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", new_callable=AsyncMock, return_value=mock_response, - ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and stubs the access assert covered by auth tests - "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", - new=AsyncMock(), - ), patch( # test-quality-ok: stubs the direct-endpoint access assert covered by auth tests - "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", - new=AsyncMock(), + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and grants access so real store resolution runs + "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", + new=AsyncMock(return_value=True), ): response = client_internal_user.post( "/v1/rag/query", @@ -424,7 +418,9 @@ def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): assert response.status_code == 200, response.json() forwarded_config = mock_aquery.await_args.kwargs["retrieval_config"] - assert forwarded_config["aws_region_name"] == "us-east-1" + assert forwarded_config["aws_region_name"] == "eu-west-1" + + EICAR = r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" INGEST_REQUEST = '{"ingest_options":{"vector_store":{"custom_llm_provider":"openai"}}}' diff --git a/tests/test_litellm/rag/test_main.py b/tests/test_litellm/rag/test_main.py index fdcdf342eea..51d03544910 100644 --- a/tests/test_litellm/rag/test_main.py +++ b/tests/test_litellm/rag/test_main.py @@ -349,6 +349,45 @@ async def test_aquery_minimal_retrieval_config_forwards_no_extras(): assert not (leaked & set(search_kwargs.keys())) +@pytest.mark.asyncio +async def test_aquery_does_not_forward_connection_override_keys_to_search(): + """ + Only allowlisted retrieval_config keys may reach the vector store search + call. Caller-controlled connection overrides (api_base, api_key, arbitrary + extras) must be dropped, otherwise a caller could redirect store + credentials to an attacker-chosen host. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): # test-quality-ok: asearch is the boundary the forwarding contract under test targets + await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={ + "vector_store_id": "bkt:idx", + "custom_llm_provider": "s3_vectors", + "aws_region_name": "eu-west-1", + "api_base": "https://attacker.example.com", + "api_key": "attacker-key", + "arbitrary_extra": "nope", + }, + mock_response="hi", + ) + + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["aws_region_name"] == "eu-west-1" + blocked = {"api_base", "api_key", "arbitrary_extra"} + assert not (blocked & set(search_kwargs.keys())) + + def test_rag_call_types_are_registered(): """ query/aquery/ingest/aingest are @client-decorated entry points, so their From 8b5ae3da9d49e08c4f6c8ca22d6f59c64c6bfae5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:15:51 -0700 Subject: [PATCH 026/175] test(vector_stores): package the suite dir to avoid test_main basename collision --- tests/test_litellm/vector_stores/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/test_litellm/vector_stores/__init__.py diff --git a/tests/test_litellm/vector_stores/__init__.py b/tests/test_litellm/vector_stores/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 8b0441a628c01f0cd6caa10176ae887c06d75fa7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:29:25 -0700 Subject: [PATCH 027/175] fix(vector_stores): block caller-supplied embedding selection params on query surfaces --- litellm/proxy/rag_endpoints/endpoints.py | 2 ++ .../proxy/vector_store_endpoints/endpoints.py | 24 ++++++++++++++ .../proxy/rag_endpoints/test_rag_endpoints.py | 24 ++++++++++++++ .../test_vector_store_endpoints.py | 32 +++++++++++++++++++ 4 files changed, 82 insertions(+) diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index d2c7d6f93ee..0ab7d99e4e4 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -42,6 +42,7 @@ from litellm.proxy.rag_endpoints.upload_security import ( ) from litellm.proxy.vector_store_endpoints.endpoints import ( build_request_data_from_managed_vector_store, + reject_caller_embedding_selection_params, ) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, @@ -716,6 +717,7 @@ async def rag_query( status_code=400, detail={"error": "retrieval_config must contain 'vector_store_id'"}, ) + reject_caller_embedding_selection_params(payload=retrieval_config, source="retrieval_config") resolved_stores: Final = await _authorize_nested_vector_store_ids( payload=retrieval_config, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 3fc6749f18a..7d64e648e08 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -29,6 +29,29 @@ from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry router: Final = APIRouter() + +BLOCKED_QUERY_EMBEDDING_SELECTION_PARAMS: Final = frozenset( + { + "embedding_model", + "litellm_embedding_model", + "litellm_embedding_config", + "litellm_credential_name", + } +) + + +def reject_caller_embedding_selection_params(payload: Mapping[str, object], source: str) -> None: + blocked: Final = sorted(BLOCKED_QUERY_EMBEDDING_SELECTION_PARAMS & payload.keys()) + if blocked: + raise HTTPException( + status_code=400, + detail={ + "error": f"'{blocked[0]}' cannot be set in {source}. " + "Embedding configuration comes from the vector store's server-side registration." + }, + ) + + ######################################################## # OpenAI Compatible Endpoints ######################################################## @@ -134,6 +157,7 @@ async def vector_store_search( ) data = await _read_request_body(request=request) + reject_caller_embedding_selection_params(payload=data, source="the search request body") data["vector_store_id"] = vector_store_id # Check for legacy vector store registry (non-managed vector stores) diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 342a4535b21..0085b6ebd36 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -421,6 +421,30 @@ def test_rag_query_store_params_win_over_user_retrieval_config(client_internal_u assert forwarded_config["aws_region_name"] == "eu-west-1" +@pytest.mark.parametrize( + "blocked_key", + ["embedding_model", "litellm_embedding_model", "litellm_embedding_config", "litellm_credential_name"], +) +def test_rag_query_rejects_caller_embedding_selection_params(client_internal_user, blocked_key): + """ + Regression: a caller must not pick the embedding model or credential used at + search time. Those resolve through the Router with the proxy's credentials, + bypassing the key's model permissions, so they may only come from the + managed store's server-side registration. + """ + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store", blocked_key: "attacker-choice"}, + }, + ) + + assert response.status_code == 400, response.json() + assert blocked_key in str(response.json()) + + EICAR = r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" INGEST_REQUEST = '{"ingest_options":{"vector_store":{"custom_llm_provider":"openai"}}}' diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index eae6f90863a..45a0221c8a6 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -3158,3 +3158,35 @@ class TestAzureAIAnalyzeNamedIndexClassification: user_api_key_dict=self._team_member("analyze", ["read"]), ) assert result is True + + +@pytest.mark.parametrize( + "blocked_key", + ["embedding_model", "litellm_embedding_model", "litellm_embedding_config", "litellm_credential_name"], +) +def test_vector_store_search_rejects_caller_embedding_selection_params(blocked_key): + """ + Regression: the search request body must not pick the embedding model or + credential used to embed the query. Those resolve through the Router with + the proxy's credentials, bypassing the key's model permissions, so they may + only come from the managed store's server-side registration. + """ + from fastapi.testclient import TestClient + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + + mock_auth = UserAPIKeyAuth(user_id="test_internal_user", user_role=LitellmUserRoles.INTERNAL_USER.value) + original_overrides = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: mock_auth + try: + client = TestClient(app) + response = client.post( + "/v1/vector_stores/s3-store/search", + json={"query": "hello", blocked_key: "attacker-choice"}, + ) + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 400, response.json() + assert blocked_key in str(response.json()) From 692f3b513ca9ac2fafb225c24d58f7ee5152eae6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:43:01 -0700 Subject: [PATCH 028/175] fix(proxy-extras): recover the v2 migration resolver from concurrent migrate deploy deadlocks Two instances racing prisma migrate deploy on one database deadlock on CREATE INDEX CONCURRENTLY: the victim gets P3018 with 40P01 and the survivor then sees the failed ledger row as P3009. Both were treated as unrecoverable, so neither instance came up. Roll the deadlocked migration's ledger row back and retry the deploy on P3018, consult the failed row's logs in _prisma_migrations to do the same on P3009, and retry a deadlock reported without a Prisma error code. Genuinely broken migrations still fail fast. --- .../litellm_proxy_extras/utils.py | 91 ++++++++++- .../tests/test_setup_database_fail_fast.py | 141 ++++++++++++++++++ 2 files changed, 228 insertions(+), 4 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index b8032dd0d28..fb948afd200 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -40,6 +40,8 @@ def _get_prisma_env() -> dict: _MIGRATION_TS_RE = re.compile(r"^(\d{14})_") +_MIGRATION_DEADLOCK_MARKER = "deadlock detected" + _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE) _SPEND_LOGS_ARTIFACT_DROP_RE = re.compile( r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE @@ -262,6 +264,48 @@ class ProxyExtrasDBManager: env=prisma_env, ) + @staticmethod + def _roll_back_migration_best_effort(migration_name: str) -> None: + """Mark a migration rolled back, tolerating a concurrent resolver + having already done it.""" + try: + ProxyExtrasDBManager._roll_back_migration(migration_name) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + pass + + @staticmethod + def _failed_migration_logs(migration_name: str) -> str: + """Logs recorded on the migration's failed _prisma_migrations row. + + P3009 stderr does not carry the original failure, so this is the only + way to tell a migration that lost a deadlock race against a concurrent + migrate deploy from one whose SQL is genuinely broken. Returns "" when + psycopg is missing, the DB is unreachable, or no failed row exists. + """ + database_url = os.getenv("DATABASE_URL") + if not database_url: + return "" + + try: + import psycopg + except ImportError: + return "" + + cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url) + try: + with psycopg.connect( + cleaned_url, connect_timeout=10, autocommit=True + ) as conn: + row = conn.execute( + "SELECT logs FROM _prisma_migrations " + "WHERE migration_name = %s AND finished_at IS NULL " + "AND rolled_back_at IS NULL", + (migration_name,), + ).fetchone() + except (psycopg.OperationalError, psycopg.DatabaseError): + return "" + return (row[0] or "") if row else "" + @staticmethod def _resolve_specific_migration(migration_name: str): """Mark a specific migration as applied""" @@ -658,7 +702,8 @@ class ProxyExtrasDBManager: v2 migration resolver (opt-in via --use_v2_migration_resolver). Runs `prisma migrate deploy` and handles standard recovery paths - (P3005 baseline, P3009/P3018 idempotent errors). Critically, it does + (P3005 baseline, P3009/P3018 idempotent errors, deadlocks against a + concurrent migrate deploy). Critically, it does NOT call `_resolve_all_migrations` — the diff-and-force recovery that caused schema thrashing when two LiteLLM versions contended for the same DB during rolling deploys. @@ -764,6 +809,22 @@ class ProxyExtrasDBManager: f"Detail: {resolve_err}" ) from resolve_err continue + if migration_match and _MIGRATION_DEADLOCK_MARKER in ( + ProxyExtrasDBManager._failed_migration_logs( + migration_match.group(1) + ) + ): + logger.info( + "Migration %s lost a deadlock race against a " + "concurrent migrate deploy, rolling its ledger " + "row back and retrying", + migration_match.group(1), + ) + ProxyExtrasDBManager._roll_back_migration_best_effort( + migration_match.group(1) + ) + time.sleep(random.randrange(5, 15)) + continue raise RuntimeError( "Database migration failed and cannot be auto-recovered. " f"Manual intervention required.\n\nPrisma error:\n{stderr}" @@ -809,11 +870,33 @@ class ProxyExtrasDBManager: ) from resolve_err continue + if migration_match and _MIGRATION_DEADLOCK_MARKER in stderr: + logger.info( + "Migration %s deadlocked against a concurrent " + "migrate deploy, rolling its ledger row back " + "and retrying", + migration_match.group(1), + ) + ProxyExtrasDBManager._roll_back_migration_best_effort( + migration_match.group(1) + ) + time.sleep(random.randrange(5, 15)) + continue + raise RuntimeError( "Database migration failed and cannot be auto-recovered. " f"Manual intervention required.\n\nPrisma error:\n{stderr}" ) from e + if _MIGRATION_DEADLOCK_MARKER in stderr: + logger.info( + "prisma migrate deploy attempt %s deadlocked against " + "a concurrent migrate deploy, retrying", + attempt + 1, + ) + time.sleep(random.randrange(5, 15)) + continue + raise RuntimeError( "Database migration failed and cannot be auto-recovered. " f"Manual intervention required.\n\nPrisma error:\n{stderr}" @@ -821,9 +904,9 @@ class ProxyExtrasDBManager: raise RuntimeError( "Database migration failed after 4 attempts (retry loop " - "exhausted by timeouts or repeated idempotent-recovery " - "continues). Check database connectivity, load, and " - "_prisma_migrations ledger state." + "exhausted by timeouts, deadlock retries, or repeated " + "idempotent-recovery continues). Check database connectivity, " + "load, and _prisma_migrations ledger state." ) finally: os.chdir(original_dir) diff --git a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py index 8d66bf872de..c4347a91dce 100644 --- a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py +++ b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py @@ -240,3 +240,144 @@ def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path): ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) assert ok is True assert resolve_called["n"] == 0, "v2 must not invoke the diff-and-force recovery" + + +_DEADLOCK_P3018_STDERR = ( + "Error: P3018\n" + "Migration name: 20260415120000_health_check_latest_per_model_index\n" + "Database error code: 40P01\n" + "deadlock detected" +) + + +def _stub_v2_env(monkeypatch, tmp_path): + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setattr( + ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None + ) + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) + (tmp_path / "schema.prisma").write_text("// stub") + monkeypatch.setattr("time.sleep", lambda _: None) + + +def _succeed_after(failures: int, stderr: str): + calls = {"n": 0} + + class _OkResult: + stdout = "Applied migration.\n" + stderr = "" + + def _run(*args, **kwargs): + if "deploy" not in args[0]: + return _OkResult() + calls["n"] += 1 + if calls["n"] <= failures: + raise subprocess.CalledProcessError( + returncode=1, cmd=args[0], stderr=stderr, output="" + ) + return _OkResult() + + return _run + + +def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path): + """v2: losing the migrate deploy deadlock race against a concurrent + instance rolls the ledger row back and retries instead of dying.""" + _stub_v2_env(monkeypatch, tmp_path) + + rolled_back = [] + monkeypatch.setattr( + ProxyExtrasDBManager, + "_roll_back_migration", + lambda name: rolled_back.append(name), + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_resolve_specific_migration", + lambda name: pytest.fail("a deadlocked migration must never be marked applied"), + ) + monkeypatch.setattr("subprocess.run", _succeed_after(1, _DEADLOCK_P3018_STDERR)) + + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert ok is True + assert rolled_back == ["20260415120000_health_check_latest_per_model_index"] + + +def test_v2_p3018_persistent_deadlock_exhausts_attempts(monkeypatch, tmp_path): + """v2: a deadlock on every attempt still fails after the retry budget.""" + _stub_v2_env(monkeypatch, tmp_path) + monkeypatch.setattr(ProxyExtrasDBManager, "_roll_back_migration", lambda name: None) + + with patch( + "subprocess.run", + side_effect=_fake_migrate_deploy_failure(1, _DEADLOCK_P3018_STDERR), + ): + with pytest.raises(RuntimeError, match="after 4 attempts"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + +def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_path): + """v2: the surviving instance sees the victim's failed ledger row as P3009. + When that row's logs show a deadlock, roll it back and retry.""" + _stub_v2_env(monkeypatch, tmp_path) + + stderr = ( + "Error: P3009\n" + "migrate found failed migrations in the target database\n" + "The `20260415120000_health_check_latest_per_model_index` migration " + "started at 2026-09-01 18:46:13 UTC failed" + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_failed_migration_logs", + lambda name: "ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock", + ) + rolled_back = [] + monkeypatch.setattr( + ProxyExtrasDBManager, + "_roll_back_migration", + lambda name: rolled_back.append(name), + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_resolve_specific_migration", + lambda name: pytest.fail("a deadlocked migration must never be marked applied"), + ) + monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr)) + + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert ok is True + assert rolled_back == ["20260415120000_health_check_latest_per_model_index"] + + +def test_v2_p3009_non_deadlock_ledger_row_still_raises(monkeypatch, tmp_path): + """v2: a failed ledger row whose logs show a real SQL error stays fatal.""" + _stub_v2_env(monkeypatch, tmp_path) + + stderr = ( + "Error: P3009\n" + "migrate found failed migrations in the target database\n" + "The `20260101000000_genuinely_broken` migration started at " + "2026-09-01 18:46:13 UTC failed" + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_failed_migration_logs", + lambda name: 'ERROR: syntax error at or near "BRKN"', + ) + + with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): + with pytest.raises(RuntimeError, match="cannot be auto-recovered"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + +def test_v2_bare_deadlock_stderr_retries(monkeypatch, tmp_path): + """v2: a deadlock reported without a Prisma error code (the advisory-lock + waiter as victim) is retried, not fatal.""" + _stub_v2_env(monkeypatch, tmp_path) + monkeypatch.setattr( + "subprocess.run", _succeed_after(1, "Database error: deadlock detected") + ) + + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert ok is True From 1eea8e283157d3c92be36749b2713607aebc9786 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:54:34 -0700 Subject: [PATCH 029/175] fix(deps): raise the tornado floor to 6.5.8 for GHSA-8423-8fgw-73vq and GHSA-wwv5-g3v4-889x --- pyproject.toml | 2 +- uv.lock | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2866e27e84c..d0f14722acd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -292,7 +292,7 @@ exclude = [ [tool.uv] constraint-dependencies = [ - "tornado>=6.5.6", + "tornado>=6.5.8", "aiohttp>=3.14.2,<4.0", "packaging>=24.0", "soupsieve>=2.8.4", diff --git a/uv.lock b/uv.lock index 27be919eea1..8bac024d49e 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-29T17:58:57.633306Z" +exclude-newer = "2026-08-29T20:52:40.322465Z" exclude-newer-span = "P3D" [manifest] @@ -25,7 +25,7 @@ constraints = [ { name = "packaging", specifier = ">=24.0" }, { name = "setuptools", specifier = ">=83.0.0" }, { name = "soupsieve", specifier = ">=2.8.4" }, - { name = "tornado", specifier = ">=6.5.6" }, + { name = "tornado", specifier = ">=6.5.8" }, ] overrides = [ { name = "cryptography", specifier = ">=50.0.0,<51.0" }, @@ -9441,19 +9441,19 @@ wheels = [ [[package]] name = "tornado" -version = "6.5.7" +version = "6.5.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/64/24/95ec527ad67b76d59299e5465b3935d05e4294b7e0290a3924b7487df30b/tornado-6.5.7.tar.gz", hash = "sha256:66c513a76cda70d53907bc27cf1447557699c2e95aa48ba27a442ff61c3ddfc2", size = 519252, upload-time = "2026-06-08T17:34:51.232Z" } +sdist = { url = "https://files.pythonhosted.org/packages/10/d3/343e5bb989d6515b1646cf3d40135d73f3d5e45339bded401b56cdac24dd/tornado-6.5.8.tar.gz", hash = "sha256:9452e1b208a8bd771e2cb1f2ff564985b9b214bdebbe622793e1799e0a6bd23f", size = 520493, upload-time = "2026-08-07T02:12:42.971Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/dc/c7043cab6fed8ae159fc1923ce829ada35c4dbd797d408a43858ffaf9639/tornado-6.5.7-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:148b2eb15c2c765a50796172c1e499649b35f30d2e3c3d3e15913cfa56bfb163", size = 448543, upload-time = "2026-06-08T17:34:38.052Z" }, - { url = "https://files.pythonhosted.org/packages/92/4f/090b1431e5a43df696feceffc268c5383cc079ecb5f08ce58f917109aafe/tornado-6.5.7-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9da38de27f1da3b78a966f0dae12b5a1ea9afe72ca805d84ff06508272ddf100", size = 446707, upload-time = "2026-06-08T17:34:39.594Z" }, - { url = "https://files.pythonhosted.org/packages/37/d8/ef374952fd5da67d4463122c2b8e5a96536ec10b4b339254c6dcde81d01c/tornado-6.5.7-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8d759e71906ee783f8867b93bf26a265743da4c1e2f4a018464c1ba019862972", size = 449774, upload-time = "2026-06-08T17:34:41.204Z" }, - { url = "https://files.pythonhosted.org/packages/35/37/d434c73f4c6e014b745b9b37085f34f40c022f007efff3d7fe65991899f3/tornado-6.5.7-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a46347a18f23fb92b396beebe0fb78f61dda0cc302445202c16203d8a18848b", size = 450745, upload-time = "2026-06-08T17:34:42.531Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2b/56b9aff361d7f1ab728a805ec7d7ea835f8807afa9f5cc690ea0e630efb9/tornado-6.5.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7778b30bef919231265e91c69963ce0f49a1e9c07ac900bbe75b19ce2575ba92", size = 450578, upload-time = "2026-06-08T17:34:43.787Z" }, - { url = "https://files.pythonhosted.org/packages/02/30/a7444fb23aa76860a14198fab96ac79f1866b0a6e19e26c4381b0938e50f/tornado-6.5.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e726f0c75da7726eec023aa62751ff8878bd2737e34fbdd33b1ae5897d2200f5", size = 449985, upload-time = "2026-06-08T17:34:45.326Z" }, - { url = "https://files.pythonhosted.org/packages/5c/42/5f0e56c01e8d9d36f4e23f367b85ae6cae0c1ecddd5e6977d8388ad27488/tornado-6.5.7-cp39-abi3-win32.whl", hash = "sha256:f8de3bf12d3efdd0cbe7c8887868198f8a91415e3f29fcf258d9b8eb7b1d9ae4", size = 451047, upload-time = "2026-06-08T17:34:46.784Z" }, - { url = "https://files.pythonhosted.org/packages/c9/a4/b393076ffb21b469eec5b328a0534cf03a3b90bfc6b1f09507cdd075d938/tornado-6.5.7-cp39-abi3-win_amd64.whl", hash = "sha256:de942f843533a039ef9fa3d9c88c7cd8a7c94553fb5ad0154270989b3d99a2c4", size = 451485, upload-time = "2026-06-08T17:34:48.248Z" }, - { url = "https://files.pythonhosted.org/packages/71/2e/7b1c769803121b809112cf9a00681c472eae1d80e32d7ec0e0bd61d0d0e1/tornado-6.5.7-cp39-abi3-win_arm64.whl", hash = "sha256:ff934fce95643af5f11efdae618eaa73d469dc588641e5c8d19295a0c65c4796", size = 450506, upload-time = "2026-06-08T17:34:49.702Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d5/007086fd8df5489338e204f65adce33fd4f21a4999dbb2b9cff2f897b5f4/tornado-6.5.8-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:cc6aa787d7cfab7c3d35189dc7a56fbd2399a569624c730c6b55b3d6531d0403", size = 449487, upload-time = "2026-08-07T02:12:28.682Z" }, + { url = "https://files.pythonhosted.org/packages/70/c8/5a24a99495903f594f6a199dd7beead1cbc0a13e2cb9102727bcaaf2a997/tornado-6.5.8-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9715b5eb79735b2bcd454ce216a9275b7c0470e64ea1bf5742f78b2f72b26eeb", size = 447649, upload-time = "2026-08-07T02:12:30.306Z" }, + { url = "https://files.pythonhosted.org/packages/6e/de/f2e733f386b85962d1b1dc82cd63d169b5b4580062b35397eac9244a41fe/tornado-6.5.8-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:547d63f450d570c14fe0e8db2cfb14c9bbd1c2503b4a6612586267955aa47b58", size = 450707, upload-time = "2026-08-07T02:12:31.95Z" }, + { url = "https://files.pythonhosted.org/packages/0b/94/20efeee9a01c141e9ac47c397f81679dfda24b32768fc4fff24e76d36c2c/tornado-6.5.8-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e2360a0ffbe145eca8af0b19cb7203d79b1a98dd4cccdd6b368f6f49c2e3808", size = 451677, upload-time = "2026-08-07T02:12:33.512Z" }, + { url = "https://files.pythonhosted.org/packages/42/ec/a96ccb8ccf0de2b7bc2c5fa1608a4803735018242e90c4882365a9fd418f/tornado-6.5.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5d242290bdf7ab3151bc1065fdd75c0dcc21cbc7b49f22a4c56329c2d6566d22", size = 451510, upload-time = "2026-08-07T02:12:35.346Z" }, + { url = "https://files.pythonhosted.org/packages/29/b5/93185859245ad3f00e62175f29607346788b696369347f0146e0421286bb/tornado-6.5.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7b94ff0e128fe0542f3bd331fb44d06260fc4ac16881545159f34ef08aad4195", size = 450917, upload-time = "2026-08-07T02:12:36.963Z" }, + { url = "https://files.pythonhosted.org/packages/97/cf/fe33cf062834487d34d1559746a4a12521033c22645b6d74d4bca702e018/tornado-6.5.8-cp39-abi3-win32.whl", hash = "sha256:67832909c4779c64942380cb5f044a5c6163d00831472d80e25e115de9917836", size = 451952, upload-time = "2026-08-07T02:12:38.512Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e1/468ad54333e92ccb62627e62cb88e5fc14a2171daa67ed47b1b8542d5b86/tornado-6.5.8-cp39-abi3-win_amd64.whl", hash = "sha256:11881db6b7c168494be2c2d12e65931451bdf7ee718535418ae1d8855dd5a0ee", size = 452391, upload-time = "2026-08-07T02:12:39.971Z" }, + { url = "https://files.pythonhosted.org/packages/ad/3e/cd5e4f06e34cde33b8ef66cf36aa2b5ad46354cc1af7d2136bbe365fee1d/tornado-6.5.8-cp39-abi3-win_arm64.whl", hash = "sha256:68a7468c7e289f8514d7d664101753903217eff1bb6822c6b5994a0b5f5bcb26", size = 451411, upload-time = "2026-08-07T02:12:41.469Z" }, ] [[package]] From 5192b2162c987f260c9c33700343a73ea4676749 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:04:12 -0700 Subject: [PATCH 030/175] fix(proxy-extras): schema-qualify the _prisma_migrations logs lookup for non-public Prisma schemas --- litellm-proxy-extras/litellm_proxy_extras/utils.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index fb948afd200..97b6b1c667c 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -292,14 +292,22 @@ class ProxyExtrasDBManager: return "" cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url) + ledger_table = psycopg.sql.SQL("{}.{}").format( + psycopg.sql.Identifier( + ProxyExtrasDBManager._prisma_schema_param(database_url) or "public" + ), + psycopg.sql.Identifier("_prisma_migrations"), + ) try: with psycopg.connect( cleaned_url, connect_timeout=10, autocommit=True ) as conn: row = conn.execute( - "SELECT logs FROM _prisma_migrations " - "WHERE migration_name = %s AND finished_at IS NULL " - "AND rolled_back_at IS NULL", + psycopg.sql.SQL( + "SELECT logs FROM {} " + "WHERE migration_name = %s AND finished_at IS NULL " + "AND rolled_back_at IS NULL" + ).format(ledger_table), (migration_name,), ).fetchone() except (psycopg.OperationalError, psycopg.DatabaseError): From 386946353ac425c6cbcac28368846dc2ab413bc2 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 14:39:15 -0700 Subject: [PATCH 031/175] fix(vertex): avoid duplicate DeepSeek OCR model namespace --- .../vertex_ai/ocr/deepseek_transformation.py | 3 ++- tests/ocr_tests/test_ocr_vertex_ai.py | 20 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py index 2603552152d..b57a87c3325 100644 --- a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py +++ b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py @@ -177,8 +177,9 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig): content_item = {"type": "image_url", "image_url": document_url} # Build DeepSeek OCR request + provider_model: Final = model if model.startswith("deepseek-ai/") else f"deepseek-ai/{model}" data: Final = { - "model": "deepseek-ai/" + model, + "model": provider_model, "messages": [{"role": "user", "content": [content_item]}], } diff --git a/tests/ocr_tests/test_ocr_vertex_ai.py b/tests/ocr_tests/test_ocr_vertex_ai.py index 1ba5b9d0883..1842eb063a5 100644 --- a/tests/ocr_tests/test_ocr_vertex_ai.py +++ b/tests/ocr_tests/test_ocr_vertex_ai.py @@ -5,9 +5,11 @@ Note: Vertex AI OCR automatically converts URLs to base64 data URIs since the Vertex AI endpoint doesn't have internet access. """ -import os import json +import os import tempfile +from typing import Final + import pytest from base_ocr_unit_tests import BaseOCRTest @@ -139,3 +141,19 @@ def test_vertex_ai_ocr_routing(): assert isinstance( deepseek_variant, VertexAIDeepSeekOCRConfig ), "DeepSeek variant should route to VertexAIDeepSeekOCRConfig" + + +@pytest.mark.parametrize("model", ("deepseek-ocr-maas", "deepseek-ai/deepseek-ocr-maas")) +def test_deepseek_request_uses_single_provider_namespace(model: str) -> None: + from litellm.llms.vertex_ai.ocr.deepseek_transformation import ( + VertexAIDeepSeekOCRConfig, + ) + + request: Final = VertexAIDeepSeekOCRConfig().transform_ocr_request( + model=model, + document={"type": "image_url", "image_url": "data:image/png;base64,AA=="}, + optional_params={}, + headers={}, + ) + + assert request.data["model"] == "deepseek-ai/deepseek-ocr-maas" From d59fcda8af69f5545a8e7c29b29d12362e2bffad Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:47:59 -0700 Subject: [PATCH 032/175] fix(rerank): adopt declared authenticating providers in arerank instead of resolving them get_llm_provider runs the OAuth device flow for github_copilot and chatgpt, so calling it on the event loop before the executor dispatch let an authenticated caller block the loop for the length of the polling window. Adopt the declared provider via declared_authenticating_provider, matching the metadata callers in utils.py, and only resolve for everything else. --- litellm/rerank_api/main.py | 19 +++++++++---- tests/test_litellm/rerank_api/test_main.py | 32 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index 597d1cfb863..37ca989b8d3 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -6,6 +6,7 @@ from typing import Any, Final, Literal import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler @@ -43,16 +44,22 @@ async def arerank( """ Async: Reranks a list of documents based on their relevance to the query """ - _custom_llm_provider: str | None = None # rebind-ok: set by the get_llm_provider unpack; read in the except + _custom_llm_provider: str | None = ( + None # rebind-ok: set by the declared-provider guard or the get_llm_provider unpack; read in the except + ) try: loop: Final = asyncio.get_event_loop() kwargs["arerank"] = True - _, _custom_llm_provider, _, _ = litellm.get_llm_provider( # rebind-ok: see pre-declaration above - model=model, - custom_llm_provider=custom_llm_provider, - api_base=kwargs.get("api_base", None), - ) + declared_provider: Final = declared_authenticating_provider(model, custom_llm_provider) + if declared_provider is not None: + _custom_llm_provider = declared_provider # rebind-ok: see pre-declaration above + else: + _, _custom_llm_provider, _, _ = litellm.get_llm_provider( # rebind-ok: see pre-declaration above + model=model, + custom_llm_provider=custom_llm_provider, + api_base=kwargs.get("api_base", None), + ) func: Final = partial( rerank, diff --git a/tests/test_litellm/rerank_api/test_main.py b/tests/test_litellm/rerank_api/test_main.py index 62149c742d6..2b6cfeda2c2 100644 --- a/tests/test_litellm/rerank_api/test_main.py +++ b/tests/test_litellm/rerank_api/test_main.py @@ -172,6 +172,38 @@ async def test_arerank_error_is_mapped_to_litellm_exception(respx_mock: respx.Mo assert "None - " not in str(exc_info.value) +@pytest.mark.asyncio +async def test_arerank_declared_authenticating_provider_skips_resolution(monkeypatch): + """Regression for the event-loop hazard in arerank's provider pre-resolution: + get_llm_provider runs the blocking OAuth device flow for github_copilot/chatgpt, + so arerank must adopt the declared provider instead of resolving it, while the + except path still maps with that declared provider.""" + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + resolution_calls = [] + + def record_resolution(*args, **kwargs): + resolution_calls.append((args, kwargs)) + return "gpt-4o", "github_copilot", None, None + + def rerank_raises_provider_error(*args, **kwargs): + raise BaseLLMException(status_code=401, message='{"error":"bad key"}') + + monkeypatch.setattr(litellm, "get_llm_provider", record_resolution) + monkeypatch.setattr("litellm.rerank_api.main.rerank", rerank_raises_provider_error) + + with pytest.raises(litellm.AuthenticationError) as exc_info: + await litellm.arerank( + model="github_copilot/gpt-4o", + query=MARKER_QUERY, + documents=[MARKER_DOC], + ) + + assert resolution_calls == [] + assert "Github_copilotException" in str(exc_info.value) + assert "None - " not in str(exc_info.value) + + @pytest.mark.asyncio async def test_together_rerank_async_honors_env_api_base(respx_mock: respx.MockRouter, monkeypatch): """Regression: TOGETHER_AI_API_BASE was honored by chat but ignored by rerank.""" From bfa5eac76b18ae9e3965d5f942b2fd0382e6b796 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 10:54:39 -0700 Subject: [PATCH 033/175] fix(vector-store): resolve embedding aliases for search --- .../proxy/vector_store_endpoints/endpoints.py | 13 ++-- .../management_endpoints.py | 48 +++++++++++--- .../test_vector_store_endpoints.py | 66 ++++++++++++------- 3 files changed, 89 insertions(+), 38 deletions(-) diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index a59d7a277cc..e0b6cf8817a 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -70,17 +70,17 @@ async def _update_request_data_with_litellm_managed_vector_store_registry( # time, instead of at row-creation time. The resolved # ``api_key`` / ``api_base`` / ``api_version`` lives only in # this per-request ``data`` dict and is never persisted. - # Legacy rows that already carry a resolved (cleartext) - # ``litellm_embedding_config`` skip the lookup and pass through - # unchanged so the embed call keeps working. + # Legacy rows that carry a resolved config are refreshed when the + # embedding model is an alias so the provider-qualified model is used. embedding_model: Final = litellm_params.get("litellm_embedding_model") - if embedding_model and not litellm_params.get("litellm_embedding_config"): + if embedding_model: from litellm.proxy.proxy_server import prisma_client - resolved_config: Final = await _resolve_embedding_config( + embedding_resolution: Final = await _resolve_embedding_config( embedding_model=embedding_model, prisma_client=prisma_client ) - if resolved_config: + if embedding_resolution: + resolved_model, resolved_config = embedding_resolution # Build a fresh dict via spread instead of mutating # ``litellm_params`` in place — the registry hands back # a reference to its cached object, so an in-place @@ -88,6 +88,7 @@ async def _update_request_data_with_litellm_managed_vector_store_registry( # in-memory cache for the lifetime of the process. litellm_params = { **litellm_params, + "litellm_embedding_model": resolved_model, "litellm_embedding_config": resolved_config, } data.update(litellm_params) diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 183a03cc13c..1930ec4aaba 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -10,7 +10,7 @@ All /vector_store management endpoints import copy import json -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, TypeAlias from fastapi import APIRouter, Depends, HTTPException @@ -49,6 +49,7 @@ from litellm.types.vector_stores import ( from litellm.vector_stores.vector_store_registry import VectorStoreRegistry router: Final = APIRouter() +EmbeddingResolution: TypeAlias = tuple[str, dict[str, object]] def _vector_store_table(prisma_client: "PrismaClient") -> "TableActions[_VectorStoreRow]": @@ -155,7 +156,19 @@ async def _fetch_and_authorize_vector_store( return typed -def _resolve_embedding_config_from_router(embedding_model: str, llm_router) -> dict[str, object] | None: +def _provider_qualified_embedding_model( + fallback: str, + model: object, + custom_llm_provider: object, +) -> str: + if not isinstance(model, str) or not model: + return fallback + if "/" in model or not isinstance(custom_llm_provider, str) or not custom_llm_provider: + return model + return f"{custom_llm_provider}/{model}" + + +def _resolve_embedding_config_from_router(embedding_model: str, llm_router) -> EmbeddingResolution | None: """ Resolve embedding config from router's config-defined models. @@ -168,7 +181,7 @@ def _resolve_embedding_config_from_router(embedding_model: str, llm_router) -> d llm_router: The LiteLLM router instance Returns: - Dictionary with api_key, api_base, and api_version if model found, None otherwise + Provider-qualified model and its connection config if found, otherwise None """ if not embedding_model or llm_router is None: return None @@ -218,12 +231,21 @@ def _resolve_embedding_config_from_router(embedding_model: str, llm_router) -> d if project_id: embedding_config["project_id"] = project_id + resolved_model: Final = _provider_qualified_embedding_model( + fallback=embedding_model, + model=getattr(litellm_params, "model", None), + custom_llm_provider=getattr(litellm_params, "custom_llm_provider", None), + ) + # Only return config if we have at least api_key or api_base if embedding_config: verbose_proxy_logger.debug( "Resolved embedding config from router model %s: %s", model_name, list(embedding_config.keys()) ) - return embedding_config + return ( + resolved_model, + embedding_config, + ) except Exception as e: verbose_proxy_logger.debug("Error resolving embedding config from router for model %s: %s", model_name, e) continue @@ -233,7 +255,7 @@ def _resolve_embedding_config_from_router(embedding_model: str, llm_router) -> d async def _resolve_embedding_config_from_db( embedding_model: str, prisma_client: "PrismaClient" -) -> dict[str, object] | None: +) -> EmbeddingResolution | None: """ Resolve embedding config from database model configuration. @@ -246,7 +268,7 @@ async def _resolve_embedding_config_from_db( prisma_client: The Prisma client instance Returns: - Dictionary with api_key, api_base, and api_version if model found, None otherwise + Provider-qualified model and its connection config if found, otherwise None """ if not embedding_model: return None @@ -315,7 +337,15 @@ async def _resolve_embedding_config_from_db( model_name, list(embedding_config.keys()), ) - return embedding_config + resolved_model: Final = _provider_qualified_embedding_model( + fallback=embedding_model, + model=decrypted_params.get("model"), + custom_llm_provider=decrypted_params.get("custom_llm_provider"), + ) + return ( + resolved_model, + embedding_config, + ) except Exception as e: verbose_proxy_logger.debug("Error resolving embedding config for model %s: %s", model_name, e) continue @@ -325,7 +355,7 @@ async def _resolve_embedding_config_from_db( async def _resolve_embedding_config( embedding_model: str, prisma_client: "PrismaClient | None", llm_router: "Router | None" = None -) -> dict[str, object] | None: +) -> EmbeddingResolution | None: """ Resolve embedding config from either router (config-defined) or database models. @@ -343,7 +373,7 @@ async def _resolve_embedding_config( llm_router: The LiteLLM router instance (optional, will be imported if not provided) Returns: - Dictionary with api_key, api_base, and api_version if model found, None otherwise + Provider-qualified model and its connection config if found, otherwise None """ if not embedding_model: return None diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index eae6f90863a..1484adb258f 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -514,7 +514,7 @@ async def test_update_request_data_resolves_embedding_config_at_use_time(): "vector_store_id": "test_store", "custom_llm_provider": "azure_ai", "litellm_params": { - "litellm_embedding_model": "azure/text-embedding-3-large", + "litellm_embedding_model": "multilingual-e5-large", # Note: no litellm_embedding_config persisted }, } @@ -534,24 +534,22 @@ async def test_update_request_data_resolves_embedding_config_at_use_time(): patch.object(litellm, "vector_store_registry", mock_registry), patch( "litellm.proxy.vector_store_endpoints.endpoints._resolve_embedding_config", - new=AsyncMock(return_value=resolved), + new=AsyncMock(return_value=("azure/multilingual-e5-large", resolved)), ), ): result = await _update_request_data_with_litellm_managed_vector_store_registry( data={}, vector_store_id="test_store" ) - assert result["litellm_embedding_model"] == "azure/text-embedding-3-large" + assert result["litellm_embedding_model"] == "azure/multilingual-e5-large" assert result["litellm_embedding_config"] == resolved @pytest.mark.asyncio -async def test_update_request_data_passes_through_legacy_embedding_config(): +async def test_update_request_data_preserves_legacy_embedding_config_when_model_not_resolved(): """A vector store row created by an older proxy version may already carry a fully-resolved ``litellm_embedding_config`` in its persisted - ``litellm_params`` (the very leak this PR closes). Those legacy rows - must still work — the use-time resolver skips re-resolution when - the config is already present so the embed call keeps succeeding.""" + ``litellm_params``. Preserve it when the model cannot be resolved.""" legacy_config = { "api_key": "legacy-cleartext-key", "api_base": "https://legacy-azure.example", @@ -571,7 +569,7 @@ async def test_update_request_data_passes_through_legacy_embedding_config(): mock_vector_store ) - resolve_mock = AsyncMock() + resolve_mock = AsyncMock(return_value=None) with ( patch.object(litellm, "vector_store_registry", mock_registry), @@ -585,7 +583,7 @@ async def test_update_request_data_passes_through_legacy_embedding_config(): ) assert result["litellm_embedding_config"] == legacy_config - resolve_mock.assert_not_awaited() + resolve_mock.assert_awaited_once() class TestCheckVectorStorePermission: @@ -2010,6 +2008,7 @@ async def test_resolve_embedding_config_from_db(): # Mock database model with litellm_params mock_db_model = MagicMock() mock_db_model.litellm_params = { + "model": "openai/text-embedding-3-small", "api_key": "test-api-key", "api_base": "https://api.openai.com", "api_version": "2024-01-01", @@ -2028,9 +2027,11 @@ async def test_resolve_embedding_config_from_db(): ) assert result is not None - assert result["api_key"] == "test-api-key" - assert result["api_base"] == "https://api.openai.com" - assert result["api_version"] == "2024-01-01" + resolved_model, resolved_config = result + assert resolved_model == "openai/text-embedding-3-small" + assert resolved_config["api_key"] == "test-api-key" + assert resolved_config["api_base"] == "https://api.openai.com" + assert resolved_config["api_version"] == "2024-01-01" mock_prisma_client.db.litellm_proxymodeltable.find_first.assert_called_once_with( where={"model_name": "text-embedding-ada-002"} ) @@ -2164,6 +2165,8 @@ def test_resolve_embedding_config_from_router(): mock_litellm_params.api_key = "config-api-key" mock_litellm_params.api_base = "https://config-api-base.com" mock_litellm_params.api_version = "2024-02-01" + mock_litellm_params.model = "text-embedding-3-small" + mock_litellm_params.custom_llm_provider = "openai" mock_deployment = MagicMock(spec=Deployment) mock_deployment.litellm_params = mock_litellm_params @@ -2176,9 +2179,11 @@ def test_resolve_embedding_config_from_router(): ) assert result is not None - assert result["api_key"] == "config-api-key" - assert result["api_base"] == "https://config-api-base.com" - assert result["api_version"] == "2024-02-01" + resolved_model, resolved_config = result + assert resolved_model == "openai/text-embedding-3-small" + assert resolved_config["api_key"] == "config-api-key" + assert resolved_config["api_base"] == "https://config-api-base.com" + assert resolved_config["api_version"] == "2024-02-01" mock_router.get_deployment_by_model_group_name.assert_called_once_with( model_group_name="text-embedding-ada-002" @@ -2197,6 +2202,8 @@ def test_resolve_embedding_config_from_router_with_provider_prefix(): mock_litellm_params.api_key = "azure-api-key" mock_litellm_params.api_base = "https://azure-endpoint.openai.azure.com" mock_litellm_params.api_version = "2024-02-15" + mock_litellm_params.model = "text-embedding-3-large" + mock_litellm_params.custom_llm_provider = "azure" mock_deployment = MagicMock(spec=Deployment) mock_deployment.litellm_params = mock_litellm_params @@ -2209,9 +2216,11 @@ def test_resolve_embedding_config_from_router_with_provider_prefix(): ) assert result is not None - assert result["api_key"] == "azure-api-key" - assert result["api_base"] == "https://azure-endpoint.openai.azure.com" - assert result["api_version"] == "2024-02-15" + resolved_model, resolved_config = result + assert resolved_model == "azure/text-embedding-3-large" + assert resolved_config["api_key"] == "azure-api-key" + assert resolved_config["api_base"] == "https://azure-endpoint.openai.azure.com" + assert resolved_config["api_version"] == "2024-02-15" # Should have tried both the full name and stripped name assert mock_router.get_deployment_by_model_group_name.call_count == 2 @@ -2239,6 +2248,8 @@ def test_resolve_embedding_config_from_router_handles_os_environ(): mock_litellm_params.api_key = "os.environ/OPENAI_API_KEY" mock_litellm_params.api_base = "https://direct-url.com" mock_litellm_params.api_version = None + mock_litellm_params.model = "text-embedding-3-small" + mock_litellm_params.custom_llm_provider = "openai" mock_deployment = MagicMock(spec=Deployment) mock_deployment.litellm_params = mock_litellm_params @@ -2254,9 +2265,11 @@ def test_resolve_embedding_config_from_router_handles_os_environ(): ) assert result is not None - assert result["api_key"] == "resolved-from-env" - assert result["api_base"] == "https://direct-url.com" - assert "api_version" not in result + resolved_model, resolved_config = result + assert resolved_model == "openai/text-embedding-3-small" + assert resolved_config["api_key"] == "resolved-from-env" + assert resolved_config["api_base"] == "https://direct-url.com" + assert "api_version" not in resolved_config mock_get_secret.assert_called_once_with("os.environ/OPENAI_API_KEY") @@ -2274,6 +2287,8 @@ async def test_resolve_embedding_config_tries_router_then_db(): mock_litellm_params.api_key = "router-api-key" mock_litellm_params.api_base = "https://router-api-base.com" mock_litellm_params.api_version = None + mock_litellm_params.model = "text-embedding-3-small" + mock_litellm_params.custom_llm_provider = "openai" mock_deployment = MagicMock(spec=Deployment) mock_deployment.litellm_params = mock_litellm_params @@ -2290,7 +2305,9 @@ async def test_resolve_embedding_config_tries_router_then_db(): ) assert result is not None - assert result["api_key"] == "router-api-key" + resolved_model, resolved_config = result + assert resolved_model == "openai/text-embedding-3-small" + assert resolved_config["api_key"] == "router-api-key" # DB should NOT have been called since router found the model mock_prisma_client.db.litellm_proxymodeltable.find_first.assert_not_called() @@ -2345,6 +2362,7 @@ async def test_resolve_embedding_config_falls_back_to_db(): # DB has the model mock_db_model = MagicMock() mock_db_model.litellm_params = { + "model": "openai/text-embedding-3-small", "api_key": "db-api-key", "api_base": "https://db-api-base.com", } @@ -2363,7 +2381,9 @@ async def test_resolve_embedding_config_falls_back_to_db(): ) assert result is not None - assert result["api_key"] == "db-api-key" + resolved_model, resolved_config = result + assert resolved_model == "openai/text-embedding-3-small" + assert resolved_config["api_key"] == "db-api-key" # DB should have been called since router didn't find the model mock_prisma_client.db.litellm_proxymodeltable.find_first.assert_called() From 5635811726ed05811abe5a242645dafd48eca9a0 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 12:15:03 -0700 Subject: [PATCH 034/175] fix(vector-store): route embeddings through router --- .../base_llm/vector_store/transformation.py | 61 +- litellm/llms/custom_httpx/llm_http_handler.py | 6 + .../valkey/vector_stores/transformation.py | 39 +- .../proxy/vector_store_endpoints/endpoints.py | 32 +- .../management_endpoints.py | 289 +-------- litellm/router.py | 56 +- litellm/vector_stores/main.py | 29 +- .../test_router_embedding_integration.py | 94 ++- .../test_valkey_transformation.py | 34 +- .../test_vector_store_endpoints.py | 576 +++++------------- uv.lock | 22 +- 11 files changed, 469 insertions(+), 769 deletions(-) diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 02a51a8bace..772e4f849a0 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -1,10 +1,14 @@ +from __future__ import annotations + from abc import abstractmethod from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, NoReturn +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, NoReturn, Protocol, runtime_checkable import httpx from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import EmbeddingResponse from litellm.types.vector_stores import ( VECTOR_STORE_OPENAI_PARAMS, BaseVectorStoreAuthCredentials, @@ -17,6 +21,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router from ..chat.transformation import BaseLLMException as _BaseLLMException @@ -27,6 +32,58 @@ else: BaseLLMException = Any +@runtime_checkable +class VectorStoreEmbeddingExecutor(Protocol): + def embed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: ... + + async def aembed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: ... + + +@dataclass(frozen=True, slots=True) +class LiteLLMVectorStoreEmbeddingExecutor: + def embed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: + import litellm + + return litellm.embedding( # pyright: ignore[reportCallIssue, reportUnknownMemberType, reportUnknownVariableType] # provider kwargs are intentionally dynamic + model=model, + input=[query], # mutable-ok: LiteLLM embedding requires a mutable input list + **dict(configuration), # pyright: ignore[reportArgumentType] # provider-specific embedding config is validated downstream # mutable-ok: kwargs require a concrete dict + ) + + async def aembed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: + import litellm + + return await litellm.aembedding( # pyright: ignore[reportUnknownMemberType] # provider kwargs are intentionally dynamic + model=model, + input=[query], # mutable-ok: LiteLLM embedding requires a mutable input list + **dict(configuration), # pyright: ignore[reportArgumentType] # provider-specific embedding config is validated downstream # mutable-ok: kwargs require a concrete dict + ) + + +@dataclass(frozen=True, slots=True) +class RouterVectorStoreEmbeddingExecutor: + router: Router + metadata: Mapping[str, object] + + def embed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: + if configuration: + return LiteLLMVectorStoreEmbeddingExecutor().embed(model, query, configuration) + return self.router.embedding( # pyright: ignore[reportUnknownMemberType] # Router embedding input retains a legacy untyped list + model=model, + input=[query], # mutable-ok: Router embedding requires a mutable input list + metadata=dict(self.metadata), # mutable-ok: Router metadata requires a concrete dict + ) + + async def aembed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: + if configuration: + return await LiteLLMVectorStoreEmbeddingExecutor().aembed(model, query, configuration) + return await self.router.aembedding( # pyright: ignore[reportUnknownMemberType] # Router embedding input retains a legacy untyped list + model=model, + input=[query], # mutable-ok: Router embedding requires a mutable input list + metadata=dict(self.metadata), # mutable-ok: Router metadata requires a concrete dict + ) + + class BaseVectorStoreConfig: def get_supported_openai_params(self, model: str) -> list[VECTOR_STORE_OPENAI_PARAMS]: return [] @@ -172,6 +229,7 @@ class BaseDirectVectorStoreConfig(BaseVectorStoreConfig): vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None = None, timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: pass @@ -184,6 +242,7 @@ class BaseDirectVectorStoreConfig(BaseVectorStoreConfig): vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None = None, timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: pass diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 834f7d564a2..118656b81a2 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -70,6 +70,7 @@ from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeech from litellm.llms.base_llm.vector_store.transformation import ( BaseDirectVectorStoreConfig, BaseVectorStoreConfig, + VectorStoreEmbeddingExecutor, ) from litellm.llms.base_llm.vector_store_files.transformation import ( BaseVectorStoreFilesConfig, @@ -9683,6 +9684,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, extra_headers: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, timeout: float | httpx.Timeout | None = None, @@ -9702,6 +9704,7 @@ class BaseLLMHTTPHandler: vector_store_search_optional_params=vector_store_search_optional_params, litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), # mutable-ok: snapshot GenericLiteLLMParams into the Mapping shape + embedding_executor=embedding_executor, timeout=timeout, ) @@ -9797,6 +9800,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, extra_headers: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, timeout: float | httpx.Timeout | None = None, @@ -9812,6 +9816,7 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + embedding_executor=embedding_executor, extra_headers=extra_headers, extra_body=extra_body, timeout=timeout, @@ -9831,6 +9836,7 @@ class BaseLLMHTTPHandler: vector_store_search_optional_params=vector_store_search_optional_params, litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), # mutable-ok: snapshot GenericLiteLLMParams into the Mapping shape + embedding_executor=embedding_executor, timeout=timeout, ) diff --git a/litellm/llms/valkey/vector_stores/transformation.py b/litellm/llms/valkey/vector_stores/transformation.py index 3cbfca0f1a9..b250f71cf3f 100644 --- a/litellm/llms/valkey/vector_stores/transformation.py +++ b/litellm/llms/valkey/vector_stores/transformation.py @@ -15,7 +15,10 @@ import httpx from pydantic import BaseModel, ConfigDict import litellm -from litellm.llms.base_llm.vector_store.transformation import BaseDirectVectorStoreConfig +from litellm.llms.base_llm.vector_store.transformation import ( + BaseDirectVectorStoreConfig, + VectorStoreEmbeddingExecutor, +) from litellm.llms.valkey.common_utils import build_valkey_url, pack_vector from litellm.types.utils import EmbeddingResponse from litellm.types.vector_stores import ( @@ -213,6 +216,7 @@ class ValkeyVectorStoreConfig(BaseDirectVectorStoreConfig): vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, litellm_logging_obj: "LiteLLMLoggingObj", litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None = None, timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: params: Final = _ValkeySearchParams.model_validate(litellm_params) @@ -222,10 +226,18 @@ class ValkeyVectorStoreConfig(BaseDirectVectorStoreConfig): embedding_field=params.embedding_field, text_field=params.text_field, ) - embedding_response: Final = self.embedding_fn( - model=params.require_embedding_model(), - input=[query_text], # mutable-ok: litellm.embedding's input contract is a list - **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + embedding_response: Final = ( + embedding_executor.embed( + params.require_embedding_model(), + query_text, + params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG, + ) + if embedding_executor is not None + else self.embedding_fn( + model=params.require_embedding_model(), + input=[query_text], # mutable-ok: the injected embedding callable requires list input + **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + ) ) vec_params: Final = {"vec": pack_vector(embedding_response.data[0]["embedding"])} # mutable-ok: redis-py API @@ -252,6 +264,7 @@ class ValkeyVectorStoreConfig(BaseDirectVectorStoreConfig): vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, litellm_logging_obj: "LiteLLMLoggingObj", litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None = None, timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: params: Final = _ValkeySearchParams.model_validate(litellm_params) @@ -261,10 +274,18 @@ class ValkeyVectorStoreConfig(BaseDirectVectorStoreConfig): embedding_field=params.embedding_field, text_field=params.text_field, ) - embedding_response: Final = await self.aembedding_fn( - model=params.require_embedding_model(), - input=[query_text], # mutable-ok: litellm.embedding's input contract is a list - **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + embedding_response: Final = ( + await embedding_executor.aembed( + params.require_embedding_model(), + query_text, + params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG, + ) + if embedding_executor is not None + else await self.aembedding_fn( + model=params.require_embedding_model(), + input=[query_text], # mutable-ok: the injected embedding callable requires list input + **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + ) ) vec_params: Final = {"vec": pack_vector(embedding_response.data[0]["embedding"])} # mutable-ok: redis-py API diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index e0b6cf8817a..3fc67181d5b 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -14,9 +14,6 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.utils import jsonify_object -from litellm.proxy.vector_store_endpoints.management_endpoints import ( - _resolve_embedding_config, -) from litellm.proxy.vector_store_endpoints.utils import ( assert_proxy_admin_for_vector_store_index_management, assert_user_can_access_vector_store, @@ -65,32 +62,9 @@ async def _update_request_data_with_litellm_managed_vector_store_registry( data["litellm_credential_name"] = vector_store_to_run.get("litellm_credential_name") if "litellm_params" in vector_store_to_run: - litellm_params = vector_store_to_run.get("litellm_params", {}) or {} - # Resolve ``litellm_embedding_config`` here, at request-handling - # time, instead of at row-creation time. The resolved - # ``api_key`` / ``api_base`` / ``api_version`` lives only in - # this per-request ``data`` dict and is never persisted. - # Legacy rows that carry a resolved config are refreshed when the - # embedding model is an alias so the provider-qualified model is used. - embedding_model: Final = litellm_params.get("litellm_embedding_model") - if embedding_model: - from litellm.proxy.proxy_server import prisma_client - - embedding_resolution: Final = await _resolve_embedding_config( - embedding_model=embedding_model, prisma_client=prisma_client - ) - if embedding_resolution: - resolved_model, resolved_config = embedding_resolution - # Build a fresh dict via spread instead of mutating - # ``litellm_params`` in place — the registry hands back - # a reference to its cached object, so an in-place - # update would persist the resolved cleartext into the - # in-memory cache for the lifetime of the process. - litellm_params = { - **litellm_params, - "litellm_embedding_model": resolved_model, - "litellm_embedding_config": resolved_config, - } + litellm_params: Final = ( + vector_store_to_run.get("litellm_params", {}) or {} + ) # mutable-ok: request execution merges persisted params into a mutable body data.update(litellm_params) return data diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 1930ec4aaba..8ca45f736ae 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -10,7 +10,7 @@ All /vector_store management endpoints import copy import json -from typing import TYPE_CHECKING, Any, Final, TypeAlias +from typing import TYPE_CHECKING, Any, Final from fastapi import APIRouter, Depends, HTTPException @@ -18,11 +18,8 @@ if TYPE_CHECKING: from prisma.models import LiteLLM_ManagedVectorStoresTable as _VectorStoreRow from litellm.proxy.utils import PrismaClient - from litellm.router import Router - import litellm from litellm._logging import verbose_proxy_logger -from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import REDACTED_BY_LITELM_STRING from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker @@ -32,13 +29,10 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user from litellm.proxy.vector_store_endpoints.utils import can_user_access_vector_store -from litellm.repositories.model_repository import ModelRepository from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import ManagedVectorStoresRepository -from litellm.secret_managers.main import get_secret from litellm.types.vector_stores import ( LiteLLM_ManagedVectorStore, LiteLLM_ManagedVectorStoreListResponse, @@ -49,7 +43,6 @@ from litellm.types.vector_stores import ( from litellm.vector_stores.vector_store_registry import VectorStoreRegistry router: Final = APIRouter() -EmbeddingResolution: TypeAlias = tuple[str, dict[str, object]] def _vector_store_table(prisma_client: "PrismaClient") -> "TableActions[_VectorStoreRow]": @@ -65,28 +58,6 @@ _LITELLM_PARAMS_MASKER: Final = SensitiveDataMasker() _REDACT_LITELLM_PARAMS_MAX_DEPTH: Final = 10 -# Use-time embedding-config resolution runs on every vector-store request -# whose persisted row carries only a model reference (the post-fix shape). -# Without a cache, that's one ``litellm_proxymodeltable.find_first`` per -# request — the no-DB-in-critical-path rule. Hold the resolved config in -# memory for a short TTL so a hot model name pays the DB lookup at most -# once per ``_EMBEDDING_CONFIG_CACHE_TTL`` seconds. Cleartext credentials -# only ever live in process memory (never persisted, never echoed in -# management responses), so the cache doesn't widen the disclosure surface. -_EMBEDDING_CONFIG_CACHE_TTL: Final = 60 -_EMBEDDING_CONFIG_CACHE_MAX_SIZE: Final = 256 -_embedding_config_cache: InMemoryCache | None = None - - -def _get_embedding_config_cache() -> InMemoryCache: - global _embedding_config_cache - if _embedding_config_cache is None: - _embedding_config_cache = InMemoryCache( - max_size_in_memory=_EMBEDDING_CONFIG_CACHE_MAX_SIZE, - default_ttl=_EMBEDDING_CONFIG_CACHE_TTL, - ) - return _embedding_config_cache - def _redact_sensitive_litellm_params(litellm_params: Any, _depth: int = 0) -> Any: """ @@ -156,264 +127,6 @@ async def _fetch_and_authorize_vector_store( return typed -def _provider_qualified_embedding_model( - fallback: str, - model: object, - custom_llm_provider: object, -) -> str: - if not isinstance(model, str) or not model: - return fallback - if "/" in model or not isinstance(custom_llm_provider, str) or not custom_llm_provider: - return model - return f"{custom_llm_provider}/{model}" - - -def _resolve_embedding_config_from_router(embedding_model: str, llm_router) -> EmbeddingResolution | None: - """ - Resolve embedding config from router's config-defined models. - - Config-defined models (from proxy_config.yaml) are stored in the router's model_list, - not in the database. This function looks up the model in the router and extracts - api_key, api_base, and api_version from the deployment's litellm_params. - - Args: - embedding_model: The embedding model string (e.g., "text-embedding-ada-002" or "azure/text-embedding-3-large") - llm_router: The LiteLLM router instance - - Returns: - Provider-qualified model and its connection config if found, otherwise None - """ - if not embedding_model or llm_router is None: - return None - - # Extract model name candidates - could be "text-embedding-ada-002" or "azure/text-embedding-3-large" - # Try exact match first, then try without provider prefix - model_name_candidates: Final = [embedding_model] - if "/" in embedding_model: - # If it has a provider prefix, also try without it - _, model_name = embedding_model.split("/", 1) - model_name_candidates.append(model_name) - - # Try to find model in router - for model_name in model_name_candidates: - try: - # Try to get deployment by model group name (model_name in config) - deployment = llm_router.get_deployment_by_model_group_name(model_group_name=model_name) - - if deployment is not None and deployment.litellm_params is not None: - litellm_params = deployment.litellm_params - - # Build embedding config from model params - embedding_config: dict[str, object] = {} - - # Extract api_key - api_key = getattr(litellm_params, "api_key", None) - if api_key: - # Handle os.environ/ prefix - if isinstance(api_key, str) and api_key.startswith("os.environ/"): - api_key = get_secret(api_key) - embedding_config["api_key"] = api_key - - # Extract api_base - api_base = getattr(litellm_params, "api_base", None) - if api_base: - # Handle os.environ/ prefix - if isinstance(api_base, str) and api_base.startswith("os.environ/"): - api_base = get_secret(api_base) - embedding_config["api_base"] = api_base - - # Extract api_version - api_version = getattr(litellm_params, "api_version", None) - if api_version: - embedding_config["api_version"] = api_version - - project_id = getattr(litellm_params, "project_id", None) - if project_id: - embedding_config["project_id"] = project_id - - resolved_model: Final = _provider_qualified_embedding_model( - fallback=embedding_model, - model=getattr(litellm_params, "model", None), - custom_llm_provider=getattr(litellm_params, "custom_llm_provider", None), - ) - - # Only return config if we have at least api_key or api_base - if embedding_config: - verbose_proxy_logger.debug( - "Resolved embedding config from router model %s: %s", model_name, list(embedding_config.keys()) - ) - return ( - resolved_model, - embedding_config, - ) - except Exception as e: - verbose_proxy_logger.debug("Error resolving embedding config from router for model %s: %s", model_name, e) - continue - - return None - - -async def _resolve_embedding_config_from_db( - embedding_model: str, prisma_client: "PrismaClient" -) -> EmbeddingResolution | None: - """ - Resolve embedding config from database model configuration. - - If litellm_embedding_model is provided but litellm_embedding_config is not, - this function looks up the model in the database and extracts api_key, api_base, - and api_version from the model's litellm_params to build the embedding config. - - Args: - embedding_model: The embedding model string (e.g., "text-embedding-ada-002" or "azure/text-embedding-3-large") - prisma_client: The Prisma client instance - - Returns: - Provider-qualified model and its connection config if found, otherwise None - """ - if not embedding_model: - return None - - # Extract model name - could be "text-embedding-ada-002" or "azure/text-embedding-3-large" - # Try to find model by exact match first, then try without provider prefix - model_name_candidates: Final = [embedding_model] - if "/" in embedding_model: - # If it has a provider prefix, also try without it - _, model_name = embedding_model.split("/", 1) - model_name_candidates.append(model_name) - - # Try to find model in database - for model_name in model_name_candidates: - try: - db_model = await ModelRepository(prisma_client).table.find_first(where={"model_name": model_name}) - - if db_model and db_model.litellm_params: - # Extract litellm_params (could be dict or JSON string) - model_params = db_model.litellm_params - if isinstance(model_params, str): # pyright: ignore[reportUnnecessaryIsInstance] # prisma Json is str - model_params = json.loads(model_params) - - # Decrypt values from database (similar to how proxy_server.py does it) - # Values stored in DB are encrypted, so we need to decrypt them first - decrypted_params = {} - if isinstance(model_params, dict): - for k, v in model_params.items(): - if isinstance(v, str): - # Decrypt value - returns original value if decryption fails or no key is set - decrypted_value = decrypt_value_helper(value=v, key=k, return_original_value=True) - decrypted_params[k] = decrypted_value - else: - decrypted_params[k] = v - else: - decrypted_params = model_params - - # Build embedding config from model params - embedding_config = {} - - # Extract api_key - api_key = decrypted_params.get("api_key") - if api_key: - # Handle os.environ/ prefix (after decryption, values may be os.environ/ prefixed) - if isinstance(api_key, str) and api_key.startswith("os.environ/"): - api_key = get_secret(api_key) - embedding_config["api_key"] = api_key - - # Extract api_base - api_base = decrypted_params.get("api_base") - if api_base: - # Handle os.environ/ prefix (after decryption, values may be os.environ/ prefixed) - if isinstance(api_base, str) and api_base.startswith("os.environ/"): - api_base = get_secret(api_base) - embedding_config["api_base"] = api_base - - # Extract api_version - api_version = decrypted_params.get("api_version") - if api_version: - embedding_config["api_version"] = api_version - - # Only return config if we have at least api_key or api_base - if embedding_config: - verbose_proxy_logger.debug( - "Resolved embedding config from database model %s: %s", - model_name, - list(embedding_config.keys()), - ) - resolved_model: Final = _provider_qualified_embedding_model( - fallback=embedding_model, - model=decrypted_params.get("model"), - custom_llm_provider=decrypted_params.get("custom_llm_provider"), - ) - return ( - resolved_model, - embedding_config, - ) - except Exception as e: - verbose_proxy_logger.debug("Error resolving embedding config for model %s: %s", model_name, e) - continue - - return None - - -async def _resolve_embedding_config( - embedding_model: str, prisma_client: "PrismaClient | None", llm_router: "Router | None" = None -) -> EmbeddingResolution | None: - """ - Resolve embedding config from either router (config-defined) or database models. - - This function first checks the router for config-defined models, then falls back - to the database. This allows users to use models defined in either location. - - Results are cached in process memory for ``_EMBEDDING_CONFIG_CACHE_TTL`` - seconds so the request-handling path doesn't hit the database on every - vector-store call. Negative results (model not found) are intentionally - not cached to avoid blocking a freshly-added model behind the TTL. - - Args: - embedding_model: The embedding model string (e.g., "text-embedding-ada-002" or "azure/text-embedding-3-large") - prisma_client: The Prisma client instance - llm_router: The LiteLLM router instance (optional, will be imported if not provided) - - Returns: - Provider-qualified model and its connection config if found, otherwise None - """ - if not embedding_model: - return None - - cache: Final = _get_embedding_config_cache() - cached: Final = cache.get_cache(embedding_model) - if cached is not None: - return cached - - # Import llm_router if not provided - if llm_router is None: - try: - from litellm.proxy.proxy_server import llm_router - except ImportError: - llm_router = None - - # First try to resolve from router (config-defined models) - if llm_router is not None: - router_config = _resolve_embedding_config_from_router(embedding_model=embedding_model, llm_router=llm_router) - if router_config: - verbose_proxy_logger.debug("Resolved embedding config from router for model %s", embedding_model) - cache.set_cache(embedding_model, router_config) - return router_config - - # Fall back to database - if prisma_client is not None: - db_config: Final = await _resolve_embedding_config_from_db( - embedding_model=embedding_model, prisma_client=prisma_client - ) - if db_config: - verbose_proxy_logger.debug("Resolved embedding config from database for model %s", embedding_model) - cache.set_cache(embedding_model, db_config) - return db_config - - verbose_proxy_logger.debug( - "Could not resolve embedding config for model %s from router or database", embedding_model - ) - return None - - ######################################################## # Helper Functions ######################################################## diff --git a/litellm/router.py b/litellm/router.py index 471a1116f44..9e0e267f21c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -84,6 +84,9 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( mask_credentials_in_payload, mask_sensitive_structure, ) +from litellm.llms.base_llm.vector_store.transformation import ( + RouterVectorStoreEmbeddingExecutor, +) from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_strategy.least_busy import LeastBusyLoggingHandler @@ -6319,6 +6322,34 @@ class Router: client: object | None = None, **kwargs, ): + if call_type == "vector_store_search": + metadata: Final = self._vector_store_request_metadata(kwargs) + provider_kwargs: Final = ( + { + "custom_llm_provider": custom_llm_provider + } # mutable-ok: provider kwargs are expanded into the request + if custom_llm_provider is not None + else MappingProxyType({}) + ) + search_kwargs: Final = { # mutable-ok: the routed request requires dynamic keyword arguments + **kwargs, + **provider_kwargs, + "_direct_vector_store_embedding_executor": RouterVectorStoreEmbeddingExecutor( + router=self, + metadata=metadata, + ), + } + model: Final = search_kwargs.get("model") + if isinstance(model, str) and model: + routed_kwargs: Final = { # mutable-ok: model must be removed before expanding routed kwargs + key: value for key, value in search_kwargs.items() if key != "model" + } + return self._generic_api_call_with_fallbacks( + model=model, + original_function=original_function, + **routed_kwargs, + ) + return original_function(**search_kwargs) return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs) return sync_wrapper @@ -6512,10 +6543,21 @@ class Router: "avector_store_update", "avector_store_delete", ): + vector_store_kwargs: Final = ( + { # mutable-ok: the async routed request requires dynamic keyword arguments + **kwargs, + "_direct_vector_store_embedding_executor": RouterVectorStoreEmbeddingExecutor( + router=self, + metadata=self._vector_store_request_metadata(kwargs), + ), + } + if call_type == "avector_store_search" + else kwargs + ) return await self._init_vector_store_api_endpoints( original_function=original_function, custom_llm_provider=custom_llm_provider, - **kwargs, + **vector_store_kwargs, ) elif call_type in ("afile_delete", "afile_content"): return await self._ageneric_api_call_with_fallbacks( @@ -6551,6 +6593,18 @@ class Router: return async_wrapper + @staticmethod + def _vector_store_request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: + litellm_metadata: Final = kwargs.get("litellm_metadata") + if isinstance(litellm_metadata, dict): + return cast( # cast-ok: isinstance validates the runtime dict boundary + "dict[str, object]", litellm_metadata + ) + metadata: Final = kwargs.get("metadata") + if isinstance(metadata, dict): + return cast("dict[str, object]", metadata) # cast-ok: isinstance validates the runtime dict boundary + return MappingProxyType({}) + async def _init_vector_store_api_endpoints( self, original_function: Callable, diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index 9b0ff71730a..89c3319ca5a 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -15,6 +15,10 @@ import litellm from litellm.constants import request_timeout from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.vector_store.transformation import ( + LiteLLMVectorStoreEmbeddingExecutor, + VectorStoreEmbeddingExecutor, +) from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( @@ -35,6 +39,14 @@ base_llm_http_handler = BaseLLMHTTPHandler() ################################################# +def _direct_vector_store_embedding_executor(value: object) -> VectorStoreEmbeddingExecutor: + if value is None: + return LiteLLMVectorStoreEmbeddingExecutor() + if isinstance(value, VectorStoreEmbeddingExecutor): + return value + raise TypeError("Invalid direct vector store embedding executor") + + def mock_vector_store_search_response( mock_results: list[VectorStoreSearchResult] | None = None, ): @@ -285,7 +297,12 @@ async def asearch( """ Async: Search a vector store for relevant chunks based on a query and file attributes filter. """ - local_vars: Final = locals() + embedding_executor: Final = _direct_vector_store_embedding_executor( + kwargs.pop("_direct_vector_store_embedding_executor", None) + ) + local_vars: Final = { # mutable-ok: exception logging requires a sanitized mutable snapshot + key: value for key, value in locals().items() if key != "embedding_executor" + } try: loop: Final = asyncio.get_event_loop() @@ -308,6 +325,7 @@ async def asearch( extra_body=extra_body, timeout=timeout, custom_llm_provider=custom_llm_provider, + _direct_vector_store_embedding_executor=embedding_executor, **kwargs, ) @@ -363,12 +381,16 @@ def search( Returns: VectorStoreSearchResponse containing the search results. """ - local_vars: Final = locals() + embedding_executor: Final = _direct_vector_store_embedding_executor( + kwargs.pop("_direct_vector_store_embedding_executor", None) + ) + local_vars: Final = { # mutable-ok: exception logging requires a sanitized mutable snapshot + key: value for key, value in locals().items() if key != "embedding_executor" + } try: litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) _is_async: Final = kwargs.pop("asearch", False) is True - # pull credentials from registry if available if litellm.vector_store_registry is not None and vector_store_id is not None: try: @@ -445,6 +467,7 @@ def search( custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, logging_obj=litellm_logging_obj, + embedding_executor=embedding_executor, extra_headers=extra_headers, extra_body=extra_body, timeout=timeout or request_timeout, diff --git a/tests/router_unit_tests/test_router_embedding_integration.py b/tests/router_unit_tests/test_router_embedding_integration.py index 75dacbaf08e..5c01587a6fe 100644 --- a/tests/router_unit_tests/test_router_embedding_integration.py +++ b/tests/router_unit_tests/test_router_embedding_integration.py @@ -5,17 +5,107 @@ These tests simulate real-world scenarios where headers and configuration need to be properly propagated through the router to the LLM API. """ -from unittest.mock import MagicMock, patch, AsyncMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest - from litellm import Router +from litellm.llms.base_llm.vector_store.transformation import ( + LiteLLMVectorStoreEmbeddingExecutor, + RouterVectorStoreEmbeddingExecutor, +) +from litellm.types.utils import EmbeddingResponse class TestRouterEmbeddingIntegration: """Integration tests for embedding with router configuration.""" + def test_vector_store_request_metadata_prefers_litellm_metadata(self): + assert Router._vector_store_request_metadata( + { + "litellm_metadata": {"user_api_key_team_id": "team-a"}, + "metadata": {"user_api_key_team_id": "team-b"}, + } + ) == {"user_api_key_team_id": "team-a"} + + assert Router._vector_store_request_metadata({"metadata": {"user_api_key_team_id": "team-b"}}) == { + "user_api_key_team_id": "team-b" + } + assert Router._vector_store_request_metadata({}) == {} + + def test_sync_vector_store_wrapper_injects_router_embedding_executor(self): + router = Router(model_list=[]) + original = MagicMock(return_value="searched") + wrapped = router.factory_function(original, call_type="vector_store_search") + + assert ( + wrapped( + vector_store_id="store", + query="query", + custom_llm_provider="valkey", + metadata={"user_api_key_team_id": "team-a"}, + ) + == "searched" + ) + + call_kwargs = original.call_args.kwargs + assert call_kwargs["custom_llm_provider"] == "valkey" + executor = call_kwargs["_direct_vector_store_embedding_executor"] + assert isinstance(executor, RouterVectorStoreEmbeddingExecutor) + assert executor.metadata == {"user_api_key_team_id": "team-a"} + + def test_sync_vector_store_wrapper_preserves_model_routing(self): + router = Router(model_list=[]) + original = MagicMock() + wrapped = router.factory_function(original, call_type="vector_store_search") + + with patch.object(router, "_generic_api_call_with_fallbacks", return_value="routed") as fallback: + assert wrapped(model="vector-alias", vector_store_id="store", query="query") == "routed" + + assert fallback.call_args.kwargs["model"] == "vector-alias" + assert fallback.call_args.kwargs["original_function"] is original + assert isinstance( + fallback.call_args.kwargs["_direct_vector_store_embedding_executor"], + RouterVectorStoreEmbeddingExecutor, + ) + + @pytest.mark.asyncio + async def test_vector_store_embedding_executors_cover_sdk_and_router_paths(self): + response = EmbeddingResponse(data=[{"embedding": [0.1], "index": 0, "object": "embedding"}]) + sdk_executor = LiteLLMVectorStoreEmbeddingExecutor() + + with ( + patch("litellm.embedding", return_value=response) as embedding, + patch("litellm.aembedding", new=AsyncMock(return_value=response)) as aembedding, + ): + assert sdk_executor.embed("openai/model", "sync", {"api_key": "explicit"}) is response + assert await sdk_executor.aembed("openai/model", "async", {"api_key": "explicit"}) is response + + embedding.assert_called_once_with(model="openai/model", input=["sync"], api_key="explicit") + aembedding.assert_awaited_once_with(model="openai/model", input=["async"], api_key="explicit") + + mock_router = MagicMock() + mock_router.embedding.return_value = response + router_executor = RouterVectorStoreEmbeddingExecutor( + router=mock_router, + metadata={"user_api_key_team_id": "team-a"}, + ) + assert router_executor.embed("team-alias", "query", {}) is response + mock_router.embedding.assert_called_once_with( + model="team-alias", + input=["query"], + metadata={"user_api_key_team_id": "team-a"}, + ) + + with patch("litellm.embedding", return_value=response) as explicit_embedding: + assert router_executor.embed("openai/model", "query", {"api_key": "store-key"}) is response + explicit_embedding.assert_called_once_with(model="openai/model", input=["query"], api_key="store-key") + mock_router.embedding.assert_called_once() + + with patch("litellm.aembedding", new=AsyncMock(return_value=response)) as explicit_aembedding: + assert await router_executor.aembed("openai/model", "query", {"api_key": "store-key"}) is response + explicit_aembedding.assert_awaited_once_with(model="openai/model", input=["query"], api_key="store-key") + def test_embedding_with_deployment_specific_headers(self): """ Test that deployment-specific headers are propagated. diff --git a/tests/test_litellm/llms/valkey/vector_stores/test_valkey_transformation.py b/tests/test_litellm/llms/valkey/vector_stores/test_valkey_transformation.py index a2ee2c2bdb1..aa114f128c5 100644 --- a/tests/test_litellm/llms/valkey/vector_stores/test_valkey_transformation.py +++ b/tests/test_litellm/llms/valkey/vector_stores/test_valkey_transformation.py @@ -67,20 +67,52 @@ class FakeAsyncEmbeddingFn(FakeEmbeddingFn): return SimpleNamespace(data=[{"embedding": self.embedding}]) +class FakeEmbeddingExecutor: + def __init__(self, embedding): + self.embedding = embedding + self.captured = None + + def embed(self, model, query, configuration): + self.captured = (model, query, configuration) + return SimpleNamespace(data=[{"embedding": self.embedding}]) + + async def aembed(self, model, query, configuration): + self.captured = (model, query, configuration) + return SimpleNamespace(data=[{"embedding": self.embedding}]) + + def _doc(doc_id, distance, **fields): return SimpleNamespace(id=doc_id, vector_distance=str(distance), **fields) -def _search(config, client=None, query="what is litellm", optional_params=None, litellm_params=None): +def _search(config, client=None, query="what is litellm", optional_params=None, litellm_params=None, executor=None): return config.execute_search_vector_store_request( vector_store_id="my_index", query=query, vector_store_search_optional_params=optional_params or {}, litellm_logging_obj=MagicMock(), litellm_params={"litellm_embedding_model": "openai/text-embedding-3-small", **(litellm_params or {})}, + embedding_executor=executor, ) +def test_sync_search_uses_request_embedding_executor_without_overwriting_explicit_config(): + executor = FakeEmbeddingExecutor([0.1, 0.2]) + config = ValkeyVectorStoreConfig(sync_client=FakeRedis()) + embedding_config = {"api_key": "store-specific-key", "aws_region_name": "us-west-2"} + + _search( + config, + litellm_params={ + "litellm_embedding_model": "team-embedding-alias", + "litellm_embedding_config": embedding_config, + }, + executor=executor, + ) + + assert executor.captured == ("team-embedding-alias", "what is litellm", embedding_config) + + def test_sync_search_builds_knn_query_with_packed_vector(): embedding_fn = FakeEmbeddingFn([0.1, 0.2, 0.3]) client = FakeRedis() diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index 1484adb258f..ad411e874ca 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -2,29 +2,24 @@ from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest -from fastapi import Request - - -from fastapi import HTTPException +from fastapi import HTTPException, Request import litellm from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( LiteLLM_ManagedVectorStore, ) +from litellm.llms.base_llm.vector_store.transformation import ( + LiteLLMVectorStoreEmbeddingExecutor, + RouterVectorStoreEmbeddingExecutor, +) from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.vector_store_endpoints.endpoints import ( _update_request_data_with_litellm_managed_vector_store_registry, index_create, index_list, ) -from litellm.proxy.vector_store_files_endpoints.endpoints import ( - _update_request_data_with_model_routing_hint, -) from litellm.proxy.vector_store_endpoints.management_endpoints import ( _check_vector_store_access, - _resolve_embedding_config, - _resolve_embedding_config_from_db, - _resolve_embedding_config_from_router, create_vector_store_in_db, new_vector_store, ) @@ -33,8 +28,12 @@ from litellm.proxy.vector_store_endpoints.utils import ( is_allowed_to_call_vector_store_endpoint, is_allowed_to_call_vector_store_files_endpoint, ) +from litellm.proxy.vector_store_files_endpoints.endpoints import ( + _update_request_data_with_model_routing_hint, +) +from litellm.types.utils import EmbeddingResponse, LlmProviders from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse -from litellm.types.utils import LlmProviders +from litellm.vector_stores.main import _direct_vector_store_embedding_executor def _serialize_litellm_params(litellm_params): @@ -51,17 +50,98 @@ def _serialize_litellm_params(litellm_params): return json.dumps(litellm_params or {}) -@pytest.fixture(autouse=True) -def _reset_embedding_config_cache(): - """The use-time embedding-config resolver caches results in process - memory across calls. Reset it before every test so the resolver - actually exercises the router/DB path under test instead of returning - a value cached by an earlier test.""" - from litellm.proxy.vector_store_endpoints import management_endpoints +def test_direct_vector_store_embedding_executor_rejects_invalid_value(): + with pytest.raises(TypeError, match="Invalid direct vector store embedding executor"): + _direct_vector_store_embedding_executor(object()) - management_endpoints._embedding_config_cache = None - yield - management_endpoints._embedding_config_cache = None + +def test_router_vector_store_search_injects_executor_and_request_metadata(): + router = litellm.Router(model_list=[]) + original = MagicMock(return_value="searched") + wrapped = router.factory_function(original, call_type="vector_store_search") + + assert ( + wrapped( + vector_store_id="store", + query="query", + custom_llm_provider="valkey", + litellm_metadata={"user_api_key_team_id": "team-a"}, + ) + == "searched" + ) + + call_kwargs = original.call_args.kwargs + assert call_kwargs["custom_llm_provider"] == "valkey" + executor = call_kwargs["_direct_vector_store_embedding_executor"] + assert isinstance(executor, RouterVectorStoreEmbeddingExecutor) + assert executor.metadata == {"user_api_key_team_id": "team-a"} + assert litellm.Router._vector_store_request_metadata({"metadata": {"user_api_key_team_id": "team-b"}}) == { + "user_api_key_team_id": "team-b" + } + assert litellm.Router._vector_store_request_metadata({}) == {} + + with patch.object( # test-quality-ok: fallback dispatch is the boundary this wrapper delegates to + router, "_generic_api_call_with_fallbacks", return_value="routed" + ) as fallback: + assert wrapped(model="vector-alias", vector_store_id="store", query="query") == "routed" + assert fallback.call_args.kwargs["model"] == "vector-alias" + assert fallback.call_args.kwargs["original_function"] is original + + create_original = MagicMock() + wrapped_create = router.factory_function(create_original, call_type="vector_store_create") + with patch.object( # test-quality-ok: fallback dispatch is the boundary this wrapper delegates to + router, "_generic_api_call_with_fallbacks", return_value="created" + ) as fallback: + assert wrapped_create(name="store") == "created" + fallback.assert_called_once_with(original_function=create_original, name="store") + + +@pytest.mark.asyncio +async def test_vector_store_embedding_executors_preserve_explicit_configuration(): + response = EmbeddingResponse(data=[{"embedding": [0.1], "index": 0, "object": "embedding"}]) + sdk_executor = LiteLLMVectorStoreEmbeddingExecutor() + + with ( + patch( # test-quality-ok: isolates SDK dispatch from external embedding providers + "litellm.embedding", return_value=response + ) as embedding, + patch( # test-quality-ok: isolates async SDK dispatch from external embedding providers + "litellm.aembedding", new=AsyncMock(return_value=response) + ) as aembedding, + ): + assert sdk_executor.embed("openai/model", "sync", {"api_key": "explicit"}) is response + assert await sdk_executor.aembed("openai/model", "async", {"api_key": "explicit"}) is response + + embedding.assert_called_once_with(model="openai/model", input=["sync"], api_key="explicit") + aembedding.assert_awaited_once_with(model="openai/model", input=["async"], api_key="explicit") + + mock_router = MagicMock() + mock_router.embedding.return_value = response + router_executor = RouterVectorStoreEmbeddingExecutor( + router=mock_router, + metadata={"user_api_key_team_id": "team-a"}, + ) + + assert router_executor.embed("team-alias", "query", {}) is response + mock_router.embedding.assert_called_once_with( + model="team-alias", + input=["query"], + metadata={"user_api_key_team_id": "team-a"}, + ) + + with ( + patch( # test-quality-ok: verifies explicit store configuration at the SDK boundary + "litellm.embedding", return_value=response + ) as explicit_embedding, + patch( # test-quality-ok: verifies async explicit store configuration at the SDK boundary + "litellm.aembedding", new=AsyncMock(return_value=response) + ) as explicit_aembedding, + ): + assert router_executor.embed("openai/model", "query", {"api_key": "store-key"}) is response + assert await router_executor.aembed("openai/model", "query", {"api_key": "store-key"}) is response + + explicit_embedding.assert_called_once_with(model="openai/model", input=["query"], api_key="store-key") + explicit_aembedding.assert_awaited_once_with(model="openai/model", input=["query"], api_key="store-key") @pytest.mark.asyncio @@ -82,10 +162,11 @@ async def test_router_avector_store_search_passes_correct_args(): } # Call router's avector_store_search - result = await router.avector_store_search( + await router.avector_store_search( vector_store_id="test_store_id", query="test query", custom_llm_provider="bedrock", + metadata={"user_api_key_team_id": "team-a"}, ) # Verify the internal method was called with correct args @@ -96,6 +177,38 @@ async def test_router_avector_store_search_passes_correct_args(): assert call_args[1]["vector_store_id"] == "test_store_id" assert call_args[1]["query"] == "test query" assert call_args[1]["custom_llm_provider"] == "bedrock" + executor = call_args[1]["_direct_vector_store_embedding_executor"] + assert isinstance(executor, RouterVectorStoreEmbeddingExecutor) + assert executor.metadata["user_api_key_team_id"] == "team-a" + + +@pytest.mark.asyncio +async def test_vector_store_embedding_executor_uses_team_scoped_router_deployment(): + router = litellm.Router( + model_list=[ + { + "model_name": "shared-embedding", + "litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "team-a-key"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "shared-embedding"}, + }, + { + "model_name": "shared-embedding", + "litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "team-b-key"}, + "model_info": {"team_id": "team-b", "team_public_model_name": "shared-embedding"}, + }, + ] + ) + executor = RouterVectorStoreEmbeddingExecutor( + router=router, + metadata={"user_api_key_team_id": "team-b"}, + ) + response = EmbeddingResponse(data=[{"embedding": [0.1], "index": 0, "object": "embedding"}]) + + with patch("litellm.aembedding", new=AsyncMock(return_value=response)) as mock_aembedding: + result = await executor.aembed("shared-embedding", "query", {}) + + assert result is response + assert mock_aembedding.await_args.kwargs["api_key"] == "team-b-key" @pytest.mark.asyncio @@ -502,89 +615,30 @@ async def test_update_request_data_with_litellm_managed_vector_store_registry(): @pytest.mark.asyncio -async def test_update_request_data_resolves_embedding_config_at_use_time(): - """When the persisted vector store row carries only a - ``litellm_embedding_model`` reference (the new behaviour after - moving the auto-resolve out of write time), the request-handling - layer must resolve the embedding config so the downstream embed - call still has ``api_key`` / ``api_base`` / ``api_version``. The - resolved config lives in this per-request data dict only — never - persisted.""" - mock_vector_store: LiteLLM_ManagedVectorStore = { +async def test_managed_vector_store_keeps_embedding_reference_and_explicit_config(): + explicit_config = {"api_key": "store-specific-key", "api_base": "https://embedding.example"} + managed_vector_store: LiteLLM_ManagedVectorStore = { "vector_store_id": "test_store", - "custom_llm_provider": "azure_ai", + "custom_llm_provider": "valkey", "litellm_params": { - "litellm_embedding_model": "multilingual-e5-large", - # Note: no litellm_embedding_config persisted + "litellm_embedding_model": "team-embedding-alias", + "litellm_embedding_config": explicit_config, }, } - mock_registry = MagicMock() - mock_registry.get_litellm_managed_vector_store_from_registry.return_value = ( - mock_vector_store - ) + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = managed_vector_store - resolved = { - "api_key": "use-time-resolved-key", - "api_base": "https://my-azure.example", - "api_version": "2024-09-01", - } - - with ( - patch.object(litellm, "vector_store_registry", mock_registry), - patch( - "litellm.proxy.vector_store_endpoints.endpoints._resolve_embedding_config", - new=AsyncMock(return_value=("azure/multilingual-e5-large", resolved)), - ), - ): + with patch.object(litellm, "vector_store_registry", mock_registry): result = await _update_request_data_with_litellm_managed_vector_store_registry( - data={}, vector_store_id="test_store" + data={}, + vector_store_id="test_store", ) - assert result["litellm_embedding_model"] == "azure/multilingual-e5-large" - assert result["litellm_embedding_config"] == resolved + assert result["litellm_embedding_model"] == "team-embedding-alias" + assert result["litellm_embedding_config"] == explicit_config + assert managed_vector_store["litellm_params"]["litellm_embedding_config"] == explicit_config -@pytest.mark.asyncio -async def test_update_request_data_preserves_legacy_embedding_config_when_model_not_resolved(): - """A vector store row created by an older proxy version may already - carry a fully-resolved ``litellm_embedding_config`` in its persisted - ``litellm_params``. Preserve it when the model cannot be resolved.""" - legacy_config = { - "api_key": "legacy-cleartext-key", - "api_base": "https://legacy-azure.example", - "api_version": "2024-01-01", - } - mock_vector_store: LiteLLM_ManagedVectorStore = { - "vector_store_id": "legacy_store", - "custom_llm_provider": "azure_ai", - "litellm_params": { - "litellm_embedding_model": "azure/text-embedding-3-large", - "litellm_embedding_config": legacy_config, - }, - } - - mock_registry = MagicMock() - mock_registry.get_litellm_managed_vector_store_from_registry.return_value = ( - mock_vector_store - ) - - resolve_mock = AsyncMock(return_value=None) - - with ( - patch.object(litellm, "vector_store_registry", mock_registry), - patch( - "litellm.proxy.vector_store_endpoints.endpoints._resolve_embedding_config", - new=resolve_mock, - ), - ): - result = await _update_request_data_with_litellm_managed_vector_store_registry( - data={}, vector_store_id="legacy_store" - ) - - assert result["litellm_embedding_config"] == legacy_config - resolve_mock.assert_awaited_once() - class TestCheckVectorStorePermission: """Test suite for check_vector_store_permission function.""" @@ -2001,60 +2055,7 @@ async def test_vector_store_update_and_list_synchronization(): @pytest.mark.asyncio -async def test_resolve_embedding_config_from_db(): - """Test that _resolve_embedding_config_from_db correctly resolves embedding config from database.""" - mock_prisma_client = MagicMock() - - # Mock database model with litellm_params - mock_db_model = MagicMock() - mock_db_model.litellm_params = { - "model": "openai/text-embedding-3-small", - "api_key": "test-api-key", - "api_base": "https://api.openai.com", - "api_version": "2024-01-01", - } - - mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock( - return_value=mock_db_model - ) - - with patch( - "litellm.proxy.vector_store_endpoints.management_endpoints.decrypt_value_helper", - side_effect=lambda value, key, return_original_value: value, - ): - result = await _resolve_embedding_config_from_db( - embedding_model="text-embedding-ada-002", prisma_client=mock_prisma_client - ) - - assert result is not None - resolved_model, resolved_config = result - assert resolved_model == "openai/text-embedding-3-small" - assert resolved_config["api_key"] == "test-api-key" - assert resolved_config["api_base"] == "https://api.openai.com" - assert resolved_config["api_version"] == "2024-01-01" - mock_prisma_client.db.litellm_proxymodeltable.find_first.assert_called_once_with( - where={"model_name": "text-embedding-ada-002"} - ) - - # Test with empty embedding_model - result_empty = await _resolve_embedding_config_from_db( - embedding_model="", prisma_client=mock_prisma_client - ) - assert result_empty is None - - # Test with model not found - mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock( - return_value=None - ) - result_not_found = await _resolve_embedding_config_from_db( - embedding_model="non-existent-model", prisma_client=mock_prisma_client - ) - assert result_not_found is None - - -@pytest.mark.asyncio -async def test_new_vector_store_auto_resolves_embedding_config(): - """Test that new_vector_store auto-resolves embedding config when embedding_model is provided but config is not.""" +async def test_new_vector_store_persists_embedding_reference_without_credentials(): import json from litellm.types.vector_stores import LiteLLM_ManagedVectorStore @@ -2071,14 +2072,6 @@ async def test_new_vector_store_auto_resolves_embedding_config(): }, } - # Mock database model lookup for embedding config resolution - mock_db_model = MagicMock() - mock_db_model.litellm_params = { - "api_key": "resolved-api-key", - "api_base": "https://api.openai.com", - "api_version": "2024-01-01", - } - # Mock user API key mock_user_api_key = MagicMock(spec=UserAPIKeyAuth) mock_user_api_key.user_role = None @@ -2089,10 +2082,6 @@ async def test_new_vector_store_auto_resolves_embedding_config(): mock_prisma_client.db.litellm_managedvectorstorestable.find_unique = AsyncMock( return_value=None # Vector store doesn't exist yet ) - mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock( - return_value=mock_db_model - ) - # Track what was passed to create captured_create_data = {} @@ -2113,280 +2102,21 @@ async def test_new_vector_store_auto_resolves_embedding_config(): mock_registry = MagicMock() mock_registry.add_vector_store_to_registry = MagicMock() - # Mock router to return None (so it falls back to DB resolution) - mock_router = MagicMock() - mock_router.get_deployment_by_model_group_name.return_value = None - with ( patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), - patch("litellm.proxy.proxy_server.llm_router", mock_router), - patch( - "litellm.proxy.vector_store_endpoints.management_endpoints.decrypt_value_helper", - side_effect=lambda value, key, return_original_value: value, - ), patch.object(litellm, "vector_store_registry", mock_registry), ): - result = await new_vector_store( - vector_store=vector_store_data, user_api_key_dict=mock_user_api_key - ) + result = await new_vector_store(vector_store=vector_store_data, user_api_key_dict=mock_user_api_key) assert result["status"] == "success" - # Auto-resolve no longer happens at create time — the persisted row - # carries only the model reference, never the resolved cleartext - # credential. Resolution now happens at request-handling time inside - # ``_update_request_data_with_litellm_managed_vector_store_registry``, - # where the resolved config lives in per-request memory and is never - # written to the database. litellm_params_json = captured_create_data.get("litellm_params") assert litellm_params_json is not None litellm_params_dict = json.loads(litellm_params_json) assert "litellm_embedding_config" not in litellm_params_dict assert litellm_params_dict["litellm_embedding_model"] == "text-embedding-ada-002" - # The response must also not echo a cleartext credential — even on - # the create response, where redaction guards against caller-supplied - # cleartext or pre-existing rows that were created by an earlier - # proxy version. response_vs = result["vector_store"] - assert "resolved-api-key" not in _serialize_litellm_params( - response_vs.get("litellm_params") - ) - - -def test_resolve_embedding_config_from_router(): - """Test that _resolve_embedding_config_from_router correctly extracts credentials from config-defined models.""" - from litellm.types.router import Deployment, LiteLLM_Params - - # Create a mock router with a model - mock_router = MagicMock() - - # Create a mock deployment with litellm_params - mock_litellm_params = MagicMock(spec=LiteLLM_Params) - mock_litellm_params.api_key = "config-api-key" - mock_litellm_params.api_base = "https://config-api-base.com" - mock_litellm_params.api_version = "2024-02-01" - mock_litellm_params.model = "text-embedding-3-small" - mock_litellm_params.custom_llm_provider = "openai" - - mock_deployment = MagicMock(spec=Deployment) - mock_deployment.litellm_params = mock_litellm_params - - mock_router.get_deployment_by_model_group_name.return_value = mock_deployment - - # Test resolution - result = _resolve_embedding_config_from_router( - embedding_model="text-embedding-ada-002", llm_router=mock_router - ) - - assert result is not None - resolved_model, resolved_config = result - assert resolved_model == "openai/text-embedding-3-small" - assert resolved_config["api_key"] == "config-api-key" - assert resolved_config["api_base"] == "https://config-api-base.com" - assert resolved_config["api_version"] == "2024-02-01" - - mock_router.get_deployment_by_model_group_name.assert_called_once_with( - model_group_name="text-embedding-ada-002" - ) - - -def test_resolve_embedding_config_from_router_with_provider_prefix(): - """Test that _resolve_embedding_config_from_router handles provider prefixes like 'azure/model-name'.""" - from litellm.types.router import Deployment, LiteLLM_Params - - # Create a mock router - mock_router = MagicMock() - - # Create a mock deployment - mock_litellm_params = MagicMock(spec=LiteLLM_Params) - mock_litellm_params.api_key = "azure-api-key" - mock_litellm_params.api_base = "https://azure-endpoint.openai.azure.com" - mock_litellm_params.api_version = "2024-02-15" - mock_litellm_params.model = "text-embedding-3-large" - mock_litellm_params.custom_llm_provider = "azure" - - mock_deployment = MagicMock(spec=Deployment) - mock_deployment.litellm_params = mock_litellm_params - - # First call with full name returns None, second call with stripped name returns deployment - mock_router.get_deployment_by_model_group_name.side_effect = [None, mock_deployment] - - result = _resolve_embedding_config_from_router( - embedding_model="azure/text-embedding-3-large", llm_router=mock_router - ) - - assert result is not None - resolved_model, resolved_config = result - assert resolved_model == "azure/text-embedding-3-large" - assert resolved_config["api_key"] == "azure-api-key" - assert resolved_config["api_base"] == "https://azure-endpoint.openai.azure.com" - assert resolved_config["api_version"] == "2024-02-15" - - # Should have tried both the full name and stripped name - assert mock_router.get_deployment_by_model_group_name.call_count == 2 - - -def test_resolve_embedding_config_from_router_returns_none_when_not_found(): - """Test that _resolve_embedding_config_from_router returns None when model is not in router.""" - mock_router = MagicMock() - mock_router.get_deployment_by_model_group_name.return_value = None - - result = _resolve_embedding_config_from_router( - embedding_model="nonexistent-model", llm_router=mock_router - ) - - assert result is None - - -def test_resolve_embedding_config_from_router_handles_os_environ(): - """Test that _resolve_embedding_config_from_router handles os.environ/ prefixed values.""" - from litellm.types.router import Deployment, LiteLLM_Params - - mock_router = MagicMock() - - mock_litellm_params = MagicMock(spec=LiteLLM_Params) - mock_litellm_params.api_key = "os.environ/OPENAI_API_KEY" - mock_litellm_params.api_base = "https://direct-url.com" - mock_litellm_params.api_version = None - mock_litellm_params.model = "text-embedding-3-small" - mock_litellm_params.custom_llm_provider = "openai" - - mock_deployment = MagicMock(spec=Deployment) - mock_deployment.litellm_params = mock_litellm_params - - mock_router.get_deployment_by_model_group_name.return_value = mock_deployment - - with patch( - "litellm.proxy.vector_store_endpoints.management_endpoints.get_secret", - return_value="resolved-from-env", - ) as mock_get_secret: - result = _resolve_embedding_config_from_router( - embedding_model="text-embedding-ada-002", llm_router=mock_router - ) - - assert result is not None - resolved_model, resolved_config = result - assert resolved_model == "openai/text-embedding-3-small" - assert resolved_config["api_key"] == "resolved-from-env" - assert resolved_config["api_base"] == "https://direct-url.com" - assert "api_version" not in resolved_config - - mock_get_secret.assert_called_once_with("os.environ/OPENAI_API_KEY") - - -@pytest.mark.asyncio -async def test_resolve_embedding_config_tries_router_then_db(): - """Test that _resolve_embedding_config tries router first, then falls back to DB.""" - from litellm.types.router import Deployment, LiteLLM_Params - - mock_prisma_client = MagicMock() - mock_router = MagicMock() - - # Router has the model - mock_litellm_params = MagicMock(spec=LiteLLM_Params) - mock_litellm_params.api_key = "router-api-key" - mock_litellm_params.api_base = "https://router-api-base.com" - mock_litellm_params.api_version = None - mock_litellm_params.model = "text-embedding-3-small" - mock_litellm_params.custom_llm_provider = "openai" - - mock_deployment = MagicMock(spec=Deployment) - mock_deployment.litellm_params = mock_litellm_params - - mock_router.get_deployment_by_model_group_name.return_value = mock_deployment - - # DB should NOT be called since router has the model - mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock() - - result = await _resolve_embedding_config( - embedding_model="text-embedding-ada-002", - prisma_client=mock_prisma_client, - llm_router=mock_router, - ) - - assert result is not None - resolved_model, resolved_config = result - assert resolved_model == "openai/text-embedding-3-small" - assert resolved_config["api_key"] == "router-api-key" - - # DB should NOT have been called since router found the model - mock_prisma_client.db.litellm_proxymodeltable.find_first.assert_not_called() - - -@pytest.mark.asyncio -async def test_resolve_embedding_config_caches_result(): - """The first lookup should hit the router/DB; subsequent lookups for - the same model name should return the cached value without touching - the router or the database.""" - from litellm.types.router import Deployment, LiteLLM_Params - - mock_prisma_client = MagicMock() - mock_router = MagicMock() - - mock_litellm_params = MagicMock(spec=LiteLLM_Params) - mock_litellm_params.api_key = "router-api-key" - mock_litellm_params.api_base = "https://router-api-base.com" - mock_litellm_params.api_version = None - - mock_deployment = MagicMock(spec=Deployment) - mock_deployment.litellm_params = mock_litellm_params - mock_router.get_deployment_by_model_group_name.return_value = mock_deployment - - first = await _resolve_embedding_config( - embedding_model="cached-model", - prisma_client=mock_prisma_client, - llm_router=mock_router, - ) - assert first is not None - assert mock_router.get_deployment_by_model_group_name.call_count == 1 - - second = await _resolve_embedding_config( - embedding_model="cached-model", - prisma_client=mock_prisma_client, - llm_router=mock_router, - ) - assert second == first - # Router (and by extension the DB) was not consulted again. - assert mock_router.get_deployment_by_model_group_name.call_count == 1 - - -@pytest.mark.asyncio -async def test_resolve_embedding_config_falls_back_to_db(): - """Test that _resolve_embedding_config falls back to DB when router doesn't have the model.""" - mock_prisma_client = MagicMock() - mock_router = MagicMock() - - # Router doesn't have the model - mock_router.get_deployment_by_model_group_name.return_value = None - - # DB has the model - mock_db_model = MagicMock() - mock_db_model.litellm_params = { - "model": "openai/text-embedding-3-small", - "api_key": "db-api-key", - "api_base": "https://db-api-base.com", - } - mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock( - return_value=mock_db_model - ) - - with patch( - "litellm.proxy.vector_store_endpoints.management_endpoints.decrypt_value_helper", - side_effect=lambda value, key, return_original_value: value, - ): - result = await _resolve_embedding_config( - embedding_model="text-embedding-ada-002", - prisma_client=mock_prisma_client, - llm_router=mock_router, - ) - - assert result is not None - resolved_model, resolved_config = result - assert resolved_model == "openai/text-embedding-3-small" - assert resolved_config["api_key"] == "db-api-key" - - # DB should have been called since router didn't find the model - mock_prisma_client.db.litellm_proxymodeltable.find_first.assert_called() + assert "api_key" not in _serialize_litellm_params(response_vs.get("litellm_params")) @pytest.mark.asyncio @@ -2445,9 +2175,7 @@ async def test_new_vector_store_auto_resolves_from_router(): } return mock_created_vector_store - mock_prisma_client.db.litellm_managedvectorstorestable.create = AsyncMock( - side_effect=mock_create - ) + mock_prisma_client.db.litellm_managedvectorstorestable.create = AsyncMock(side_effect=mock_create) mock_registry = MagicMock() mock_registry.add_vector_store_to_registry = MagicMock() diff --git a/uv.lock b/uv.lock index 27be919eea1..8d886044083 100644 --- a/uv.lock +++ b/uv.lock @@ -9441,19 +9441,19 @@ wheels = [ [[package]] name = "tornado" -version = "6.5.7" +version = "6.5.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/64/24/95ec527ad67b76d59299e5465b3935d05e4294b7e0290a3924b7487df30b/tornado-6.5.7.tar.gz", hash = "sha256:66c513a76cda70d53907bc27cf1447557699c2e95aa48ba27a442ff61c3ddfc2", size = 519252, upload-time = "2026-06-08T17:34:51.232Z" } +sdist = { url = "https://files.pythonhosted.org/packages/10/d3/343e5bb989d6515b1646cf3d40135d73f3d5e45339bded401b56cdac24dd/tornado-6.5.8.tar.gz", hash = "sha256:9452e1b208a8bd771e2cb1f2ff564985b9b214bdebbe622793e1799e0a6bd23f", size = 520493, upload-time = "2026-08-07T02:12:42.971Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/dc/c7043cab6fed8ae159fc1923ce829ada35c4dbd797d408a43858ffaf9639/tornado-6.5.7-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:148b2eb15c2c765a50796172c1e499649b35f30d2e3c3d3e15913cfa56bfb163", size = 448543, upload-time = "2026-06-08T17:34:38.052Z" }, - { url = "https://files.pythonhosted.org/packages/92/4f/090b1431e5a43df696feceffc268c5383cc079ecb5f08ce58f917109aafe/tornado-6.5.7-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9da38de27f1da3b78a966f0dae12b5a1ea9afe72ca805d84ff06508272ddf100", size = 446707, upload-time = "2026-06-08T17:34:39.594Z" }, - { url = "https://files.pythonhosted.org/packages/37/d8/ef374952fd5da67d4463122c2b8e5a96536ec10b4b339254c6dcde81d01c/tornado-6.5.7-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8d759e71906ee783f8867b93bf26a265743da4c1e2f4a018464c1ba019862972", size = 449774, upload-time = "2026-06-08T17:34:41.204Z" }, - { url = "https://files.pythonhosted.org/packages/35/37/d434c73f4c6e014b745b9b37085f34f40c022f007efff3d7fe65991899f3/tornado-6.5.7-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a46347a18f23fb92b396beebe0fb78f61dda0cc302445202c16203d8a18848b", size = 450745, upload-time = "2026-06-08T17:34:42.531Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2b/56b9aff361d7f1ab728a805ec7d7ea835f8807afa9f5cc690ea0e630efb9/tornado-6.5.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7778b30bef919231265e91c69963ce0f49a1e9c07ac900bbe75b19ce2575ba92", size = 450578, upload-time = "2026-06-08T17:34:43.787Z" }, - { url = "https://files.pythonhosted.org/packages/02/30/a7444fb23aa76860a14198fab96ac79f1866b0a6e19e26c4381b0938e50f/tornado-6.5.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e726f0c75da7726eec023aa62751ff8878bd2737e34fbdd33b1ae5897d2200f5", size = 449985, upload-time = "2026-06-08T17:34:45.326Z" }, - { url = "https://files.pythonhosted.org/packages/5c/42/5f0e56c01e8d9d36f4e23f367b85ae6cae0c1ecddd5e6977d8388ad27488/tornado-6.5.7-cp39-abi3-win32.whl", hash = "sha256:f8de3bf12d3efdd0cbe7c8887868198f8a91415e3f29fcf258d9b8eb7b1d9ae4", size = 451047, upload-time = "2026-06-08T17:34:46.784Z" }, - { url = "https://files.pythonhosted.org/packages/c9/a4/b393076ffb21b469eec5b328a0534cf03a3b90bfc6b1f09507cdd075d938/tornado-6.5.7-cp39-abi3-win_amd64.whl", hash = "sha256:de942f843533a039ef9fa3d9c88c7cd8a7c94553fb5ad0154270989b3d99a2c4", size = 451485, upload-time = "2026-06-08T17:34:48.248Z" }, - { url = "https://files.pythonhosted.org/packages/71/2e/7b1c769803121b809112cf9a00681c472eae1d80e32d7ec0e0bd61d0d0e1/tornado-6.5.7-cp39-abi3-win_arm64.whl", hash = "sha256:ff934fce95643af5f11efdae618eaa73d469dc588641e5c8d19295a0c65c4796", size = 450506, upload-time = "2026-06-08T17:34:49.702Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d5/007086fd8df5489338e204f65adce33fd4f21a4999dbb2b9cff2f897b5f4/tornado-6.5.8-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:cc6aa787d7cfab7c3d35189dc7a56fbd2399a569624c730c6b55b3d6531d0403", size = 449487, upload-time = "2026-08-07T02:12:28.682Z" }, + { url = "https://files.pythonhosted.org/packages/70/c8/5a24a99495903f594f6a199dd7beead1cbc0a13e2cb9102727bcaaf2a997/tornado-6.5.8-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9715b5eb79735b2bcd454ce216a9275b7c0470e64ea1bf5742f78b2f72b26eeb", size = 447649, upload-time = "2026-08-07T02:12:30.306Z" }, + { url = "https://files.pythonhosted.org/packages/6e/de/f2e733f386b85962d1b1dc82cd63d169b5b4580062b35397eac9244a41fe/tornado-6.5.8-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:547d63f450d570c14fe0e8db2cfb14c9bbd1c2503b4a6612586267955aa47b58", size = 450707, upload-time = "2026-08-07T02:12:31.95Z" }, + { url = "https://files.pythonhosted.org/packages/0b/94/20efeee9a01c141e9ac47c397f81679dfda24b32768fc4fff24e76d36c2c/tornado-6.5.8-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e2360a0ffbe145eca8af0b19cb7203d79b1a98dd4cccdd6b368f6f49c2e3808", size = 451677, upload-time = "2026-08-07T02:12:33.512Z" }, + { url = "https://files.pythonhosted.org/packages/42/ec/a96ccb8ccf0de2b7bc2c5fa1608a4803735018242e90c4882365a9fd418f/tornado-6.5.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5d242290bdf7ab3151bc1065fdd75c0dcc21cbc7b49f22a4c56329c2d6566d22", size = 451510, upload-time = "2026-08-07T02:12:35.346Z" }, + { url = "https://files.pythonhosted.org/packages/29/b5/93185859245ad3f00e62175f29607346788b696369347f0146e0421286bb/tornado-6.5.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7b94ff0e128fe0542f3bd331fb44d06260fc4ac16881545159f34ef08aad4195", size = 450917, upload-time = "2026-08-07T02:12:36.963Z" }, + { url = "https://files.pythonhosted.org/packages/97/cf/fe33cf062834487d34d1559746a4a12521033c22645b6d74d4bca702e018/tornado-6.5.8-cp39-abi3-win32.whl", hash = "sha256:67832909c4779c64942380cb5f044a5c6163d00831472d80e25e115de9917836", size = 451952, upload-time = "2026-08-07T02:12:38.512Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e1/468ad54333e92ccb62627e62cb88e5fc14a2171daa67ed47b1b8542d5b86/tornado-6.5.8-cp39-abi3-win_amd64.whl", hash = "sha256:11881db6b7c168494be2c2d12e65931451bdf7ee718535418ae1d8855dd5a0ee", size = 452391, upload-time = "2026-08-07T02:12:39.971Z" }, + { url = "https://files.pythonhosted.org/packages/ad/3e/cd5e4f06e34cde33b8ef66cf36aa2b5ad46354cc1af7d2136bbe365fee1d/tornado-6.5.8-cp39-abi3-win_arm64.whl", hash = "sha256:68a7468c7e289f8514d7d664101753903217eff1bb6822c6b5994a0b5f5bcb26", size = 451411, upload-time = "2026-08-07T02:12:41.469Z" }, ] [[package]] From 6805d01709f9401f36bba163dc7f9c3192643c51 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 14:05:12 -0700 Subject: [PATCH 035/175] fix(vector-store): preserve aliases with embedding config --- .../base_llm/vector_store/transformation.py | 21 +++++--- .../test_router_embedding_integration.py | 53 ++++++++++++++++--- .../test_vector_store_endpoints.py | 17 +++++- 3 files changed, 75 insertions(+), 16 deletions(-) diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 772e4f849a0..e9c925448a8 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -3,7 +3,7 @@ from __future__ import annotations from abc import abstractmethod from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, NoReturn, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, runtime_checkable import httpx @@ -65,22 +65,29 @@ class RouterVectorStoreEmbeddingExecutor: router: Router metadata: Mapping[str, object] + def _embedding_kwargs(self, configuration: Mapping[str, object]) -> dict[str, object]: + configured_metadata: Final = configuration.get("metadata") + metadata: Final = { + **(configured_metadata if isinstance(configured_metadata, Mapping) else {}), + **self.metadata, + } + return { + **{key: value for key, value in configuration.items() if key not in ("input", "metadata", "model")}, + "metadata": metadata, + } + def embed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: - if configuration: - return LiteLLMVectorStoreEmbeddingExecutor().embed(model, query, configuration) return self.router.embedding( # pyright: ignore[reportUnknownMemberType] # Router embedding input retains a legacy untyped list model=model, input=[query], # mutable-ok: Router embedding requires a mutable input list - metadata=dict(self.metadata), # mutable-ok: Router metadata requires a concrete dict + **self._embedding_kwargs(configuration), # pyright: ignore[reportArgumentType] # provider kwargs are intentionally dynamic ) async def aembed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: - if configuration: - return await LiteLLMVectorStoreEmbeddingExecutor().aembed(model, query, configuration) return await self.router.aembedding( # pyright: ignore[reportUnknownMemberType] # Router embedding input retains a legacy untyped list model=model, input=[query], # mutable-ok: Router embedding requires a mutable input list - metadata=dict(self.metadata), # mutable-ok: Router metadata requires a concrete dict + **self._embedding_kwargs(configuration), # pyright: ignore[reportArgumentType] # provider kwargs are intentionally dynamic ) diff --git a/tests/router_unit_tests/test_router_embedding_integration.py b/tests/router_unit_tests/test_router_embedding_integration.py index 5c01587a6fe..d5e0c750d88 100644 --- a/tests/router_unit_tests/test_router_embedding_integration.py +++ b/tests/router_unit_tests/test_router_embedding_integration.py @@ -97,14 +97,53 @@ class TestRouterEmbeddingIntegration: metadata={"user_api_key_team_id": "team-a"}, ) - with patch("litellm.embedding", return_value=response) as explicit_embedding: - assert router_executor.embed("openai/model", "query", {"api_key": "store-key"}) is response - explicit_embedding.assert_called_once_with(model="openai/model", input=["query"], api_key="store-key") - mock_router.embedding.assert_called_once() + alias_router = Router( + model_list=[ + { + "model_name": "team-alias", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "deployment-key", + }, + } + ] + ) + alias_executor = RouterVectorStoreEmbeddingExecutor( + router=alias_router, + metadata={"user_api_key_team_id": "team-a"}, + ) + explicit_config = { + "api_base": "https://embedding.example/v1", + "api_key": "store-key", + "metadata": { + "configured": True, + "user_api_key_team_id": "untrusted-team", + }, + "model": "untrusted-model", + } - with patch("litellm.aembedding", new=AsyncMock(return_value=response)) as explicit_aembedding: - assert await router_executor.aembed("openai/model", "query", {"api_key": "store-key"}) is response - explicit_aembedding.assert_awaited_once_with(model="openai/model", input=["query"], api_key="store-key") + with ( + patch("litellm.embedding", return_value=response) as explicit_embedding, + patch("litellm.aembedding", new=AsyncMock(return_value=response)) as explicit_aembedding, + ): + assert alias_executor.embed("team-alias", "sync query", explicit_config) is response + assert await alias_executor.aembed("team-alias", "async query", explicit_config) is response + + sync_kwargs = explicit_embedding.call_args.kwargs + assert sync_kwargs["model"] == "openai/text-embedding-3-small" + assert sync_kwargs["input"] == ["sync query"] + assert sync_kwargs["api_base"] == "https://embedding.example/v1" + assert sync_kwargs["api_key"] == "store-key" + assert sync_kwargs["metadata"]["configured"] is True + assert sync_kwargs["metadata"]["user_api_key_team_id"] == "team-a" + + async_kwargs = explicit_aembedding.await_args.kwargs + assert async_kwargs["model"] == "openai/text-embedding-3-small" + assert async_kwargs["input"] == ["async query"] + assert async_kwargs["api_base"] == "https://embedding.example/v1" + assert async_kwargs["api_key"] == "store-key" + assert async_kwargs["metadata"]["configured"] is True + assert async_kwargs["metadata"]["user_api_key_team_id"] == "team-a" def test_embedding_with_deployment_specific_headers(self): """ diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index ad411e874ca..903d5cb55f3 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -117,6 +117,7 @@ async def test_vector_store_embedding_executors_preserve_explicit_configuration( mock_router = MagicMock() mock_router.embedding.return_value = response + mock_router.aembedding = AsyncMock(return_value=response) router_executor = RouterVectorStoreEmbeddingExecutor( router=mock_router, metadata={"user_api_key_team_id": "team-a"}, @@ -140,8 +141,20 @@ async def test_vector_store_embedding_executors_preserve_explicit_configuration( assert router_executor.embed("openai/model", "query", {"api_key": "store-key"}) is response assert await router_executor.aembed("openai/model", "query", {"api_key": "store-key"}) is response - explicit_embedding.assert_called_once_with(model="openai/model", input=["query"], api_key="store-key") - explicit_aembedding.assert_awaited_once_with(model="openai/model", input=["query"], api_key="store-key") + explicit_embedding.assert_not_called() + explicit_aembedding.assert_not_awaited() + assert mock_router.embedding.call_args.kwargs == { + "model": "openai/model", + "input": ["query"], + "api_key": "store-key", + "metadata": {"user_api_key_team_id": "team-a"}, + } + mock_router.aembedding.assert_awaited_once_with( + model="openai/model", + input=["query"], + api_key="store-key", + metadata={"user_api_key_team_id": "team-a"}, + ) @pytest.mark.asyncio From 5799a32cdda6647d2f16460d79b8610dbae49d34 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 15:37:50 -0700 Subject: [PATCH 036/175] fix(vector-store): route pre-call searches through router --- .../vector_store_pre_call_hook.py | 22 ++++++++-- .../test_bedrock_knowledgebase_hook.py | 42 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index 07d4f959489..aaf5cb080dc 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -80,10 +80,15 @@ class VectorStorePreCallHook(CustomLogger): # Get prisma_client for database fallback prisma_client = None + llm_router = None try: - from litellm.proxy.proxy_server import prisma_client as _prisma_client + from litellm.proxy.proxy_server import ( + llm_router as _llm_router, + prisma_client as _prisma_client, + ) prisma_client = _prisma_client + llm_router = _llm_router except ImportError: pass @@ -114,12 +119,23 @@ class VectorStorePreCallHook(CustomLogger): vector_store_id = vector_store_to_run.get("vector_store_id", "") custom_llm_provider = vector_store_to_run.get("custom_llm_provider") litellm_params_for_vector_store = vector_store_to_run.get("litellm_params", {}) or {} - # Call litellm.vector_stores.search() with the required parameters - search_response = await litellm.vector_stores.asearch( + request_litellm_params: Final = ( + litellm_logging_obj.model_call_details.get("litellm_params", {}) + if litellm_logging_obj is not None + else {} + ) + request_metadata: Final = ( + request_litellm_params.get("metadata", {}) if isinstance(request_litellm_params, dict) else {} + ) + search_function: Final = ( + llm_router.avector_store_search if llm_router is not None else litellm.vector_stores.asearch + ) + search_response = await search_function( **{ "vector_store_id": vector_store_id, "query": query, "custom_llm_provider": custom_llm_provider, + "metadata": request_metadata, **litellm_params_for_vector_store, }, ) diff --git a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py index 3f9f2bacdd3..06083b77e84 100644 --- a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py +++ b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py @@ -71,6 +71,48 @@ def setup_vector_store_registry(): ) +@pytest.mark.asyncio +async def test_vector_store_hook_routes_search_through_proxy_router( + setup_vector_store_registry, +): + proxy_router = Mock() + proxy_router.avector_store_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", + search_query="what is litellm?", + data=[ + VectorStoreSearchResult( + score=1.0, + content=[VectorStoreResultContent(text="routed context", type="text")], + ) + ], + ) + ) + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_params": {"metadata": {"user_api_key_team_id": "team-a"}} + } + + with patch("litellm.proxy.proxy_server.llm_router", proxy_router): + _, messages, _ = await VectorStorePreCallHook().async_get_chat_completion_prompt( + model="chat-model", + messages=[{"role": "user", "content": "what is litellm?"}], + non_default_params={"vector_store_ids": ["T37J8R4WTM"]}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + litellm_logging_obj=logging_obj, + ) + + proxy_router.avector_store_search.assert_awaited_once_with( + vector_store_id="T37J8R4WTM", + query="what is litellm?", + custom_llm_provider="bedrock", + metadata={"user_api_key_team_id": "team-a"}, + ) + assert messages[0]["content"] == "Context:\n\nrouted context\n\n" + + @pytest.mark.asyncio async def test_e2e_bedrock_knowledgebase_retrieval_with_completion( setup_vector_store_registry, From 0cc0c47f8b2a4196beb4e766bcc65fe84743e9ed Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 15:52:57 -0700 Subject: [PATCH 037/175] style(vector-store): satisfy import lint --- .../vector_store_pre_call_hook.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index aaf5cb080dc..e012d35b8f3 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -5,6 +5,7 @@ This hook is called before making an LLM request when a vector store is configur It searches the vector store for relevant context and appends it to the messages. """ +from collections.abc import Awaitable, Callable from typing import TYPE_CHECKING, Any, Final, cast import litellm @@ -84,6 +85,8 @@ class VectorStorePreCallHook(CustomLogger): try: from litellm.proxy.proxy_server import ( llm_router as _llm_router, + ) + from litellm.proxy.proxy_server import ( prisma_client as _prisma_client, ) @@ -119,17 +122,20 @@ class VectorStorePreCallHook(CustomLogger): vector_store_id = vector_store_to_run.get("vector_store_id", "") custom_llm_provider = vector_store_to_run.get("custom_llm_provider") litellm_params_for_vector_store = vector_store_to_run.get("litellm_params", {}) or {} - request_litellm_params: Final = ( - litellm_logging_obj.model_call_details.get("litellm_params", {}) - if litellm_logging_obj is not None - else {} - ) - request_metadata: Final = ( + request_litellm_params = litellm_logging_obj.model_call_details.get("litellm_params", {}) + request_metadata = ( request_litellm_params.get("metadata", {}) if isinstance(request_litellm_params, dict) else {} ) - search_function: Final = ( - llm_router.avector_store_search if llm_router is not None else litellm.vector_stores.asearch - ) + if llm_router is not None: + search_function = cast( # cast-ok: normalize router search callable + Callable[..., Awaitable[VectorStoreSearchResponse]], + llm_router.avector_store_search, + ) + else: + search_function = cast( # cast-ok: normalize SDK search callable + Callable[..., Awaitable[VectorStoreSearchResponse]], + litellm.vector_stores.asearch, + ) search_response = await search_function( **{ "vector_store_id": vector_store_id, From 4a68abfd49eea3b7d4fdfeaadd55598369848ef2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:26:23 +0000 Subject: [PATCH 038/175] fix(proxy): route container create and list through model_list deployments Container create and list requests had no container ID to decode, so the router called the provider handler directly and the OpenAI transformation fell back to the global OPENAI_API_KEY. Proxies configured only with model_list credentials sent Authorization: Bearer None. Route through _ageneric_api_call_with_fallbacks when the caller passes a model, expose the list endpoint's model query param to the router, and encode the managed container ID on the async create path so follow-up calls route to the same deployment. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/containers/main.py | 25 +++++--- .../proxy/container_endpoints/endpoints.py | 2 +- litellm/router.py | 11 +++- .../test_router_endpoints.py | 59 +++++++++++++++++++ .../containers/test_container_api.py | 36 +++++++++++ 5 files changed, 122 insertions(+), 11 deletions(-) diff --git a/litellm/containers/main.py b/litellm/containers/main.py index 97ca11872c1..90d59af009f 100644 --- a/litellm/containers/main.py +++ b/litellm/containers/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars import json -from collections.abc import Coroutine, Mapping +from collections.abc import Callable, Coroutine, Mapping from functools import partial from typing import Final, Literal, overload @@ -47,6 +47,13 @@ __all__ = [ ##### Container Create ####################### +async def _encode_created_container_id( + pending: Coroutine[object, object, ContainerObject], + encode: Callable[[ContainerObject], ContainerObject], +) -> ContainerObject: + return encode(await pending) + + @client async def acreate_container( name: str, @@ -256,16 +263,16 @@ def create_container( _is_async=_is_async, ) - # Encode container_id with provider/model metadata for routing + encode: Final = partial( + ContainerRequestUtils.encode_container_id_in_response, + custom_llm_provider=custom_llm_provider, + litellm_metadata=kwargs.get("litellm_metadata"), + extra_body=extra_body, + ) if isinstance(container_obj, ContainerObject): - container_obj = ContainerRequestUtils.encode_container_id_in_response( - response_obj=container_obj, - custom_llm_provider=custom_llm_provider, - litellm_metadata=kwargs.get("litellm_metadata"), - extra_body=extra_body, - ) + return encode(container_obj) - return container_obj + return _encode_created_container_id(pending=container_obj, encode=encode) except Exception as e: raise litellm.exception_type( diff --git a/litellm/proxy/container_endpoints/endpoints.py b/litellm/proxy/container_endpoints/endpoints.py index 4a088140725..eaa3db336a9 100644 --- a/litellm/proxy/container_endpoints/endpoints.py +++ b/litellm/proxy/container_endpoints/endpoints.py @@ -208,7 +208,7 @@ async def list_containers( # Read query parameters query_params: Final = dict(request.query_params) - data: Final[dict[str, Any]] = {"query_params": query_params} + data: Final[dict[str, Any]] = {"query_params": query_params, "model": query_params.get("model")} # Extract custom_llm_provider using priority chain custom_llm_provider: Final = ( diff --git a/litellm/router.py b/litellm/router.py index 471a1116f44..52e858a0b6c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6589,7 +6589,9 @@ class Router: metadata. When present, decode the ID, replace ``container_id`` with the upstream value, and route through ``_ageneric_api_call_with_fallbacks`` so deployment credentials (e.g. regional ``api_base`` for Azure) match - :meth:`_init_responses_api_endpoints`. Otherwise call the handler directly. + :meth:`_init_responses_api_endpoints`. Create/list calls carry no container ID, so + they route through the deployment named by ``model`` when the caller passes one. + Otherwise call the handler directly with global provider credentials. """ if custom_llm_provider and "custom_llm_provider" not in kwargs: kwargs["custom_llm_provider"] = custom_llm_provider @@ -6621,6 +6623,13 @@ class Router: **kwargs, ) + requested_model: Final = kwargs.get("model") + if isinstance(requested_model, str) and requested_model.strip(): + return await self._ageneric_api_call_with_fallbacks( + original_function=original_function, + **kwargs, + ) + return await original_function(**kwargs) async def _init_responses_api_endpoints( diff --git a/tests/router_unit_tests/test_router_endpoints.py b/tests/router_unit_tests/test_router_endpoints.py index d37af5b456a..6fcbd77e054 100644 --- a/tests/router_unit_tests/test_router_endpoints.py +++ b/tests/router_unit_tests/test_router_endpoints.py @@ -1324,6 +1324,65 @@ async def test_init_containers_api_endpoints_managed_id_without_model_id_applies assert call_kw["custom_llm_provider"] == "azure" +@pytest.mark.asyncio +async def test_init_containers_api_endpoints_create_with_model_uses_deployment_credentials(monkeypatch): + """ + ``POST /v1/containers`` carries no container ID, so a ``model`` in the request + body is the only way to pick a deployment. The upstream call must receive that + deployment's ``api_key``/``api_base`` instead of falling back to the global + ``OPENAI_API_KEY`` (which may be unset on the proxy). + """ + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + router = Router( + model_list=[ + { + "model_name": "gpt-5.4", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_key": "sk-model-list-key", + "api_base": "https://custom.openai.example/v1", + }, + } + ] + ) + mock_original_function = AsyncMock(return_value={"id": "cntr_test", "name": "Test Container"}) + + await router._init_containers_api_endpoints( + original_function=mock_original_function, + custom_llm_provider="openai", + name="Test Container", + model="gpt-5.4", + ) + + mock_original_function.assert_called_once() + call_kw = mock_original_function.call_args.kwargs + assert call_kw["api_key"] == "sk-model-list-key" + assert call_kw["api_base"] == "https://custom.openai.example/v1" + assert call_kw["model"] == "openai/gpt-5.4" + assert call_kw["name"] == "Test Container" + + +@pytest.mark.asyncio +async def test_init_containers_api_endpoints_create_without_model_calls_directly(): + """ + Without ``model`` (or with ``model=None`` as the proxy forwards it), create/list + must keep calling the handler directly with global provider credentials. + """ + router = Router(model_list=[]) + router._ageneric_api_call_with_fallbacks = AsyncMock() + mock_original_function = AsyncMock(return_value={"id": "cntr_test"}) + + await router._init_containers_api_endpoints( + original_function=mock_original_function, + custom_llm_provider="openai", + name="Test Container", + model=None, + ) + + router._ageneric_api_call_with_fallbacks.assert_not_called() + mock_original_function.assert_called_once_with(custom_llm_provider="openai", name="Test Container", model=None) + + def test_router_model_group_encrypted_content_affinity_callback_registration(): from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, diff --git a/tests/test_litellm/containers/test_container_api.py b/tests/test_litellm/containers/test_container_api.py index 885c4cd294a..8f1d87f9603 100644 --- a/tests/test_litellm/containers/test_container_api.py +++ b/tests/test_litellm/containers/test_container_api.py @@ -152,6 +152,42 @@ class TestContainerAPI: assert response.id == "cntr_async_123" assert response.name == "Async Test Container" + @pytest.mark.asyncio + async def test_acreate_container_encodes_router_model_id(self): + """ + The async handler returns a coroutine, so the managed-ID encoding must run + after it resolves. Otherwise follow-up calls (retrieve/delete/files) lose the + deployment and fall back to global provider credentials. + """ + upstream_response = ContainerObject( + id="cntr_upstream_123", + object="container", + created_at=1747857508, + status="running", + expires_after={"anchor": "last_active_at", "minutes": 20}, + last_active_at=1747857508, + name="Routed Container", + ) + + async def _resolve_upstream(): + return upstream_response + + with patch.object( # test-quality-ok: create_container does not forward a client, so the handler is the only seam + base_llm_http_handler, + "container_create_handler", + side_effect=lambda **kwargs: _resolve_upstream() if kwargs["_is_async"] else upstream_response, + ): + response = await acreate_container( + name="Routed Container", + custom_llm_provider="openai", + litellm_metadata={"model_info": {"id": "deployment-abc"}}, + ) + + decoded = ResponsesAPIRequestUtils._decode_container_id(response.id) + assert decoded["model_id"] == "deployment-abc" + assert decoded["custom_llm_provider"] == "openai" + assert decoded["response_id"] == "cntr_upstream_123" + @pytest.mark.asyncio async def test_alist_containers_basic(self): """Test basic async container listing functionality.""" From acc65b27d210158a18db74d6d0cd48d12aeb3a40 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:39:42 +0000 Subject: [PATCH 039/175] fix(router): pass container create/list through when model names no deployment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 6 ++-- .../test_router_endpoints.py | 33 +++++++++++++++++++ .../containers/test_container_api.py | 2 +- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 52e858a0b6c..7b471a69ecf 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6590,8 +6590,9 @@ class Router: upstream value, and route through ``_ageneric_api_call_with_fallbacks`` so deployment credentials (e.g. regional ``api_base`` for Azure) match :meth:`_init_responses_api_endpoints`. Create/list calls carry no container ID, so - they route through the deployment named by ``model`` when the caller passes one. - Otherwise call the handler directly with global provider credentials. + they route through the deployment named by ``model`` when the caller passes one, + falling back to the direct call when no deployment matches. Otherwise call the + handler directly with global provider credentials. """ if custom_llm_provider and "custom_llm_provider" not in kwargs: kwargs["custom_llm_provider"] = custom_llm_provider @@ -6627,6 +6628,7 @@ class Router: if isinstance(requested_model, str) and requested_model.strip(): return await self._ageneric_api_call_with_fallbacks( original_function=original_function, + passthrough_on_no_deployment=True, **kwargs, ) diff --git a/tests/router_unit_tests/test_router_endpoints.py b/tests/router_unit_tests/test_router_endpoints.py index 6fcbd77e054..a06aaa363ad 100644 --- a/tests/router_unit_tests/test_router_endpoints.py +++ b/tests/router_unit_tests/test_router_endpoints.py @@ -1383,6 +1383,39 @@ async def test_init_containers_api_endpoints_create_without_model_calls_directly mock_original_function.assert_called_once_with(custom_llm_provider="openai", name="Test Container", model=None) +@pytest.mark.asyncio +async def test_init_containers_api_endpoints_create_with_unknown_model_passes_through(monkeypatch): + """ + A ``model`` that names no configured deployment must not turn into a 400. The call + falls through to the handler with the caller's model and no injected deployment + credentials, matching the behaviour before model-based routing existed. + """ + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + router = Router( + model_list=[ + { + "model_name": "gpt-5.4", + "litellm_params": {"model": "openai/gpt-5.4", "api_key": "sk-model-list-key"}, + } + ] + ) + mock_original_function = AsyncMock(return_value={"id": "cntr_test"}) + + await router._init_containers_api_endpoints( + original_function=mock_original_function, + custom_llm_provider="openai", + name="Test Container", + model="does-not-exist", + ) + + mock_original_function.assert_called_once() + call_kw = mock_original_function.call_args.kwargs + assert call_kw["model"] == "does-not-exist" + assert call_kw["name"] == "Test Container" + assert "api_key" not in call_kw + assert "api_base" not in call_kw + + def test_router_model_group_encrypted_content_affinity_callback_registration(): from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, diff --git a/tests/test_litellm/containers/test_container_api.py b/tests/test_litellm/containers/test_container_api.py index 8f1d87f9603..16a3844431e 100644 --- a/tests/test_litellm/containers/test_container_api.py +++ b/tests/test_litellm/containers/test_container_api.py @@ -172,7 +172,7 @@ class TestContainerAPI: async def _resolve_upstream(): return upstream_response - with patch.object( # test-quality-ok: create_container does not forward a client, so the handler is the only seam + with patch.object( # test-quality-ok: create_container exposes no client seam, only the handler base_llm_http_handler, "container_create_handler", side_effect=lambda **kwargs: _resolve_upstream() if kwargs["_is_async"] else upstream_response, From fcd9052179f039f075b3e25a8c1aec8657fc98a0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:10:52 -0700 Subject: [PATCH 040/175] feat(proxy): honor model_info.display_name in the Anthropic-shaped /v1/models listing --- litellm/llms/anthropic/common_utils.py | 17 ++-- .../proxy/common_utils/model_listing_utils.py | 25 +++++- litellm/proxy/proxy_server.py | 21 +++-- litellm/router.py | 20 +++++ .../proxy/proxy_server/test_routes_models.py | 78 +++++++++++++++++++ .../test_team_model_name_translation.py | 26 ++++++- tests/test_litellm/test_router.py | 65 ++++++++++++++++ 7 files changed, 240 insertions(+), 12 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index c60ebd844ba..d23690976ad 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1378,31 +1378,38 @@ def process_anthropic_headers(headers: httpx.Headers | dict) -> dict: return additional_headers -def _anthropic_model_entry(model: ModelInfoResponse, created_at: str) -> Mapping[str, object]: +def _anthropic_model_entry( + model: ModelInfoResponse, created_at: str, display_names: Mapping[str, str] +) -> Mapping[str, object]: return { # mutable-ok: JSON response body, serialized by the route and never mutated "type": "model", "id": model["id"], - "display_name": model["id"], + "display_name": display_names.get(model["id"], model["id"]), "created_at": created_at, "max_input_tokens": model.get("max_input_tokens"), "max_tokens": model.get("max_output_tokens"), } -def create_anthropic_model_list_response(models: Sequence[ModelInfoResponse]) -> Mapping[str, object]: +def create_anthropic_model_list_response( + models: Sequence[ModelInfoResponse], + display_names: Mapping[str, str] = MappingProxyType({}), +) -> Mapping[str, object]: """Build the Anthropic-native /v1/models envelope. Clients that send an anthropic-version header parse the Anthropic Models API shape (type/display_name/created_at plus has_more/first_id/last_id) and filter the list themselves, so every model is returned here. The token limits carry over from the OpenAI-shaped listing, named as the Messages API names them, and - are always present because the vendor shape declares them nullable, not optional + are always present because the vendor shape declares them nullable, not optional. + display_names maps a listed model id to a configured human-readable name; ids + without an entry fall back to the id itself, matching the vendor behavior """ created_at: Final = ( datetime.fromtimestamp(DEFAULT_MODEL_CREATED_AT_TIME, tz=timezone.utc).isoformat().replace("+00:00", "Z") ) data: Final = [ # mutable-ok: JSON response body, serialized by the route and never mutated - _anthropic_model_entry(model, created_at) for model in models + _anthropic_model_entry(model, created_at, display_names) for model in models ] return { # mutable-ok: JSON response body, serialized by the route and never mutated "data": data, diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index 9fd24162f7e..213a697b3dd 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -10,13 +10,36 @@ legacy internal names with `general_settings.use_team_public_model_name: false`. from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast if TYPE_CHECKING: from litellm.router import Router +def configured_display_names( + entries: Sequence[tuple[str, str]], + llm_router: Router | None, +) -> Mapping[str, str]: + """response_id -> configured `model_info.display_name` for the listing entries + that have one. + + Metadata is looked up by each entry's internal lookup id (so team-scoped rows + resolve), while the returned map is keyed by the public response id the + Anthropic-shaped listing is built from. Entries without a configured name are + omitted so the listing falls back to the id itself. + """ + if llm_router is None: + return MappingProxyType({}) + resolved: Final = ( + (response_id, llm_router.get_configured_display_name(lookup_id)) for response_id, lookup_id in entries + ) + return MappingProxyType( + {response_id: display_name for response_id, display_name in resolved if display_name is not None} + ) + + class TeamModelNameTranslator: """Translates internal team routing keys to their public names for the model listing/retrieve responses. Stateless; the live router and general_settings diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2c600667283..4f172ca29b9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -351,7 +351,10 @@ from litellm.proxy.common_utils.load_config_utils import ( get_file_contents_from_s3, ) from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations -from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator +from litellm.proxy.common_utils.model_listing_utils import ( + TeamModelNameTranslator, + configured_display_names, +) from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, ) @@ -10193,7 +10196,8 @@ async def model_list( # The internal routing key drives the metadata/fallback lookup, while the # public name is what the client sees as the model id. model_data = [] - for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings): + admin_entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings) + for response_id, lookup_id in admin_entries: model_info = create_model_info_response( model_id=lookup_id, provider="openai", @@ -10206,7 +10210,10 @@ async def model_list( if wants_anthropic_format: admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above - return create_anthropic_model_list_response(admin_listing) + return create_anthropic_model_list_response( + admin_listing, + display_names=configured_display_names(admin_entries, llm_router), + ) return dict( data=model_data, @@ -10237,7 +10244,8 @@ async def model_list( # The internal routing key drives the metadata/fallback lookup, while the # public name is what the client sees as the model id. model_data = [] - for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings): + entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings) + for response_id, lookup_id in entries: model_info = create_model_info_response( model_id=lookup_id, provider="openai", @@ -10250,7 +10258,10 @@ async def model_list( if wants_anthropic_format: listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above - return create_anthropic_model_list_response(listing) + return create_anthropic_model_list_response( + listing, + display_names=configured_display_names(entries, llm_router), + ) return dict( data=model_data, diff --git a/litellm/router.py b/litellm/router.py index 471a1116f44..6245f8a6a03 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9607,6 +9607,26 @@ class Router: coerce_token_limit(model_info.get("max_output_tokens")), ) + def get_configured_display_name(self, model_name: str) -> "str | None": + """ + Return the display_name explicitly configured in a concrete deployment's + model_info for model_name, via O(1) index lookup. + + Returns None for wildcard-expanded or unknown names, and treats a + non-string or empty configured value as absent rather than failing the + listing. Like get_configured_token_limits, this never triggers pattern + matching or deep copies, so it is safe to call per listed model on the + /v1/models hot path. + """ + deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return None + + display_name: Final = deployment.model_info.get("display_name") + if isinstance(display_name, str) and display_name.strip(): + return display_name + return None + def get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None ) -> dict[str, Any] | None: diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_models.py b/tests/test_litellm/proxy/proxy_server/test_routes_models.py index 2b126b1ea95..bc6106a06f8 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_models.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_models.py @@ -45,6 +45,7 @@ def patched_models(monkeypatch): deployment = MagicMock() deployment.litellm_params.model = "gpt-4" router.get_deployment_by_model_group_name = MagicMock(return_value=deployment) + router.get_configured_display_name = MagicMock(return_value=None) monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) @@ -187,6 +188,83 @@ def test_anthropic_format_carries_router_configured_token_limits(client, auth_as assert (claude["max_input_tokens"], claude["max_tokens"]) == (500000, 4096) +@pytest.mark.parametrize("path", ["/v1/models", "/models"]) +def test_anthropic_format_uses_configured_display_name(client, auth_as, patched_models, path): + """A deployment's ``model_info.display_name`` becomes the Anthropic-native + ``display_name`` so Claude Code's picker shows a clean name while the id keeps + routing; models without one keep the id fallback, and the OpenAI-shaped + listing carries no display_name either way.""" + + def _configured(model_name): + return "Kimi K3" if model_name == "gpt-4" else None + + patched_models.get_configured_display_name = MagicMock(side_effect=_configured) + + with auth_as(): + anthropic_response = client.get(path, headers={"anthropic-version": "2023-06-01"}) + openai_response = client.get(path) + + assert anthropic_response.status_code == 200 + gpt_4, claude = anthropic_response.json()["data"] + assert (gpt_4["id"], gpt_4["display_name"]) == ("gpt-4", "Kimi K3") + assert (claude["id"], claude["display_name"]) == ("claude-sonnet", "claude-sonnet") + + assert openai_response.status_code == 200 + openai_models = openai_response.json()["data"] + assert [m["id"] for m in openai_models] == ["gpt-4", "claude-sonnet"] + assert all("display_name" not in m for m in openai_models) + + +@pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) +def test_anthropic_display_name_resolved_via_internal_team_key( + client, auth_as, patched_models, monkeypatch, params +): + """For a team-scoped row the configured display name must be looked up by the + internal routing key while the entry itself is keyed by the public name, so + the clean name lands on the id the client actually sees.""" + from litellm.proxy import utils as proxy_utils + from litellm.proxy.auth import model_checks + + internal_name = "model_name_team-1_c0ffee" + + patched_models.get_model_list = MagicMock( + return_value=[ + { + "model_name": internal_name, + "model_info": { + "team_id": "team-1", + "team_public_model_name": "gpt-4-team", + }, + } + ] + ) + patched_models.get_model_names = MagicMock(return_value=[internal_name]) + patched_models.get_configured_display_name = MagicMock( + side_effect=lambda model_name: "Team GPT" if model_name == internal_name else None + ) + + async def _fake_get_available_models_for_user(**kwargs): + return [internal_name] + + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + _fake_get_available_models_for_user, + ) + monkeypatch.setattr( + model_checks, "get_complete_model_list", lambda **kwargs: [internal_name] + ) + + with auth_as(): + response = client.get( + "/v1/models", params=params, headers={"anthropic-version": "2023-06-01"} + ) + + assert response.status_code == 200 + (entry,) = response.json()["data"] + assert (entry["id"], entry["display_name"]) == ("gpt-4-team", "Team GPT") + + @pytest.mark.parametrize("path", ["/v1/models", "/models"]) def test_get_models_invalid_scope_returns_400(client, auth_as, patched_models, path): """Pins: ``GET /v1/models``, ``GET /models`` (error path: invalid scope).""" diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index aa35fd64f18..0fb9b1a6d88 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -19,7 +19,10 @@ from litellm.proxy._types import ( LitellmUserRoles, UserAPIKeyAuth, ) -from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator +from litellm.proxy.common_utils.model_listing_utils import ( + TeamModelNameTranslator, + configured_display_names, +) from litellm.proxy.proxy_server import ( _get_proxy_model_info, _translate_model_name_for_response, @@ -1391,6 +1394,27 @@ def test_resolve_public_name_respects_legacy_flag(): ) +def test_configured_display_names_keyed_by_response_id(): + """The map is keyed by the public response id while the router lookup uses + the internal routing key, and entries without a configured name are omitted.""" + router = MagicMock() + router.get_configured_display_name = MagicMock( + side_effect=lambda model_name: "Team Sonnet" if model_name == "model_name_team-abc-123_4a6b8" else None + ) + + assert configured_display_names( + entries=[ + ("team-claude-sonnet", "model_name_team-abc-123_4a6b8"), + ("gpt-4o", "gpt-4o"), + ], + llm_router=router, + ) == {"team-claude-sonnet": "Team Sonnet"} + + +def test_configured_display_names_empty_without_router(): + assert configured_display_names(entries=[("gpt-4o", "gpt-4o")], llm_router=None) == {} + + @pytest.mark.asyncio async def test_retrieve_model_by_public_name_returns_200(monkeypatch): """Regression: `GET /v1/models/{public_name}` must NOT 404. The listing diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 44c1cdbff06..f4ea9b03a80 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7271,6 +7271,71 @@ def test_get_configured_token_limits_coerces_numeric_strings(): assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000) +def test_get_configured_display_name_reads_deployment_model_info(): + router = litellm.Router( + model_list=[ + { + "model_name": "Kimi K3-claude-compatible", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"display_name": "Kimi K3"}, + } + ] + ) + + assert router.get_configured_display_name("Kimi K3-claude-compatible") == "Kimi K3" + + +def test_get_configured_display_name_returns_none_for_unset_or_unknown(): + router = litellm.Router( + model_list=[ + { + "model_name": "no-display-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + } + ] + ) + + assert router.get_configured_display_name("no-display-model") is None + assert router.get_configured_display_name("not-a-real-model") is None + + +def test_get_configured_display_name_skips_wildcard_pattern_matching(): + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock/*", + "litellm_params": {"model": "bedrock/*"}, + "model_info": {"display_name": "Bedrock"}, + } + ] + ) + + with patch.object( + router.pattern_router, "route", side_effect=AssertionError("pattern route called") + ): + assert ( + router.get_configured_display_name("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") + is None + ) + + +def test_get_configured_display_name_treats_malformed_values_as_absent(): + malformed = ["", " ", 12345, ["Kimi K3"], {"name": "Kimi K3"}, True] + router = litellm.Router( + model_list=[ + { + "model_name": f"bad-display-{i}", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"display_name": bad}, + } + for i, bad in enumerate(malformed) + ] + ) + + for i in range(len(malformed)): + assert router.get_configured_display_name(f"bad-display-{i}") is None + + @pytest.mark.asyncio async def test_acreate_batch_disable_fallbacks_surfaces_owning_provider_error(): router = litellm.Router( From 1cd99a036e0538bb61b280e17639545c75374d81 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 1 Sep 2026 17:29:58 -0700 Subject: [PATCH 041/175] fix(router): route Claude Code subagents through session router --- litellm/router.py | 80 +++++++++++++++++- tests/test_litellm/test_router.py | 129 ++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 6e4405ebfef..b49269f2457 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -353,6 +353,8 @@ _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" +_CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") +_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600 def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool: @@ -12546,6 +12548,77 @@ class Router: return None return candidates[0] + @staticmethod + def _request_header(request_kwargs: Mapping[str, object], header_name: str) -> str | None: + proxy_server_request: Final = request_kwargs.get("proxy_server_request") + if not isinstance(proxy_server_request, Mapping): + return None + headers: Final = proxy_server_request.get("headers") + if not isinstance(headers, Mapping): + return None + return next( + ( + value + for key, value in headers.items() + if isinstance(key, str) and key.lower() == header_name and isinstance(value, str) + ), + None, + ) + + def _claude_code_session_router_cache_key(self, request_kwargs: Mapping[str, object]) -> str | None: + session_id: Final = self._request_header(request_kwargs, "x-claude-code-session-id") + if session_id is None or _CLAUDE_CODE_SESSION_ID_RE.fullmatch(session_id) is None: + return None + metadata_name: Final = "litellm_metadata" if "litellm_metadata" in request_kwargs else "metadata" + metadata: Final = request_kwargs.get(metadata_name) + if not isinstance(metadata, Mapping): + return None + caller_scope: Final = metadata.get("user_api_key_hash") + if not isinstance(caller_scope, str) or not caller_scope: + return None + return f"claude_code_session_router:v1:{caller_scope}:{session_id}" + + async def _resolve_claude_code_session_router( + self, + model: str, + registered_model_name: str, + request_kwargs: Mapping[str, object], + ) -> str: + cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs) + if cache_key is None or not isinstance(request_kwargs, dict): + return registered_model_name + + agent_id: Final = self._request_header(request_kwargs, "x-claude-code-agent-id") + if agent_id is not None: + bound_model: Final = await self.cache.async_get_cache(key=cache_key) + if not isinstance(bound_model, str): + return registered_model_name + bound_registered_model: Final = self._get_model_from_alias(model=bound_model) or bound_model + if self._select_pre_routing_strategy(bound_registered_model, request_kwargs) is None: + await self.cache.async_delete_cache(key=cache_key) + return registered_model_name + await self.cache.async_set_cache( + key=cache_key, + value=bound_model, + ttl=_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS, + ) + self._stamp_or_clear_metadata_key(request_kwargs, "model_group", bound_model) + return bound_registered_model + + if self._request_header(request_kwargs, "x-app") != "cli": + return registered_model_name + if request_kwargs.get("fallback_depth") not in (None, 0): + return registered_model_name + if self._select_pre_routing_strategy(registered_model_name, request_kwargs) is None: + await self.cache.async_delete_cache(key=cache_key) + return registered_model_name + await self.cache.async_set_cache( + key=cache_key, + value=model, + ttl=_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS, + ) + return registered_model_name + async def async_pre_routing_hook( self, model: str, @@ -12565,7 +12638,12 @@ class Router: the alias, since spend metadata is stamped before routing and the response carries the tier group the strategy picked. """ - registered_model_name: Final = self._get_model_from_alias(model=model) or model + requested_registered_model_name: Final = self._get_model_from_alias(model=model) or model + registered_model_name: Final = await self._resolve_claude_code_session_router( + model=model, + registered_model_name=requested_registered_model_name, + request_kwargs=request_kwargs, + ) ######################################################### # Run the routing-plugin pipeline, if any plugins are configured. diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 84f6344be35..d22a1cc04ca 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8316,6 +8316,135 @@ class TestConsumedRequestTagsStamp: assert CONSUMED_REQUEST_TAGS_METADATA_KEY not in request_kwargs["metadata"] +class TestClaudeCodeSubagentSessionRouterBinding: + class _RewriteStrategy: + async def async_pre_routing_hook( + self, model, request_kwargs, messages=None, input=None, specific_deployment=False + ): + from litellm.types.router import PreRoutingHookResponse + + return PreRoutingHookResponse( + model="cheap-model", + messages=messages, + routing_decision={ + "router_model_name": "smart-router", + "router_type": "complexity", + "routed_model": "cheap-model", + "cause": "heuristic_scorer", + }, + ) + + @classmethod + def _router(cls) -> "litellm.Router": + from litellm.types.router import TaggedPreRoutingStrategy + + router = litellm.Router( + model_list=[ + { + "model_name": "cheap-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "cheap response"}, + }, + { + "model_name": "expensive-model", + "litellm_params": {"model": "openai/gpt-4o", "mock_response": "expensive response"}, + }, + ] + ) + router.complexity_routers = { + "smart-router": [TaggedPreRoutingStrategy(tags=(), strategy=cls._RewriteStrategy())] + } + return router + + @staticmethod + def _request_kwargs( + *, + key_hash: str = "key-hash-a", + app: str = "cli", + agent_id: str | None = None, + fallback_depth: int | None = None, + ) -> dict: + headers = { + "X-Claude-Code-Session-Id": "session-1234", + "x-app": app, + **({"x-claude-code-agent-id": agent_id} if agent_id is not None else {}), + } + return { + "metadata": {"user_api_key_hash": key_hash}, + "proxy_server_request": {"headers": headers}, + **({"fallback_depth": fallback_depth} if fallback_depth is not None else {}), + } + + @pytest.mark.asyncio + async def test_subagent_concrete_model_uses_the_main_sessions_router(self): + router = self._router() + + await router.acompletion( + model="smart-router", + messages=[{"role": "user", "content": "main turn"}], + **self._request_kwargs(), + ) + subagent_kwargs = self._request_kwargs(agent_id="agent-1234") + + response = await router.acompletion( + model="expensive-model", + messages=[{"role": "user", "content": "subagent turn"}], + **subagent_kwargs, + ) + + assert response.choices[0].message.content == "cheap response" + assert subagent_kwargs["metadata"]["model_group"] == "smart-router" + assert subagent_kwargs["metadata"]["routing_decision"]["router_model_name"] == "smart-router" + + @pytest.mark.asyncio + async def test_main_direct_model_clears_the_session_router(self): + router = self._router() + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=self._request_kwargs()) + await router.async_pre_routing_hook(model="expensive-model", request_kwargs=self._request_kwargs()) + + response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(agent_id="agent-1234"), + ) + + assert response is None + + @pytest.mark.asyncio + async def test_background_and_fallback_requests_do_not_clear_the_session_router(self): + router = self._router() + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=self._request_kwargs()) + await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(app="cli-bg"), + ) + await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(fallback_depth=1), + ) + + response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(agent_id="agent-1234"), + ) + + assert response is not None + assert response.model == "cheap-model" + + @pytest.mark.asyncio + async def test_session_router_binding_is_scoped_to_the_authenticated_key(self): + router = self._router() + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=self._request_kwargs()) + + response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(key_hash="key-hash-b", agent_id="agent-1234"), + ) + + assert response is None + + class TestAutoRouterMaxInputCharsWiring: """`auto_router_max_input_chars` on the deployment has to reach the AutoRouter that embeds prompts. From 82d046c42821ca3b24036dd57e362252564c0596 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 1 Sep 2026 17:43:48 -0700 Subject: [PATCH 042/175] fix(router): make Claude session cleanup best effort --- litellm/router.py | 14 ++++++++++++-- tests/test_litellm/test_router.py | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index b49269f2457..2bb725b0880 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12578,6 +12578,16 @@ class Router: return None return f"claude_code_session_router:v1:{caller_scope}:{session_id}" + async def _delete_claude_code_session_router_binding(self, cache_key: str) -> None: + try: + await self.cache.async_delete_cache(key=cache_key) + except Exception as e: # noqa: BLE001 # cache cleanup must not fail an otherwise routable request + verbose_router_logger.warning( + "Failed to delete Claude Code session router binding; " + "the binding may remain until its TTL expires: %s", + e, + ) + async def _resolve_claude_code_session_router( self, model: str, @@ -12595,7 +12605,7 @@ class Router: return registered_model_name bound_registered_model: Final = self._get_model_from_alias(model=bound_model) or bound_model if self._select_pre_routing_strategy(bound_registered_model, request_kwargs) is None: - await self.cache.async_delete_cache(key=cache_key) + await self._delete_claude_code_session_router_binding(cache_key) return registered_model_name await self.cache.async_set_cache( key=cache_key, @@ -12610,7 +12620,7 @@ class Router: if request_kwargs.get("fallback_depth") not in (None, 0): return registered_model_name if self._select_pre_routing_strategy(registered_model_name, request_kwargs) is None: - await self.cache.async_delete_cache(key=cache_key) + await self._delete_claude_code_session_router_binding(cache_key) return registered_model_name await self.cache.async_set_cache( key=cache_key, diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index d22a1cc04ca..77b73a2d12a 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8409,6 +8409,25 @@ class TestClaudeCodeSubagentSessionRouterBinding: assert response is None + @pytest.mark.asyncio + async def test_redis_cleanup_failure_does_not_reject_a_direct_model_request(self): + from litellm.caching.caching import RedisCache + + router = self._router() + redis_cache = MagicMock(spec=RedisCache) + redis_cache.async_delete_cache = AsyncMock(side_effect=ConnectionError("redis unavailable")) + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=self._request_kwargs()) + router._update_redis_cache(cache=redis_cache) + + response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(), + ) + + assert response is None + redis_cache.async_delete_cache.assert_awaited_once() + @pytest.mark.asyncio async def test_background_and_fallback_requests_do_not_clear_the_session_router(self): router = self._router() From ac19d0dbdf61a3e2707b03d2deed225ba9d68389 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:44:29 -0700 Subject: [PATCH 043/175] fix(spend): keep every-deployment scope on gateway cache-injection marks The caching-savings marker litellm_gateway_injected_cache credits gateway-earned prompt-caching savings to the deployment it names, or to every deployment via the empty-string sentinel. Two paths lost that scope: - the router prompt-management factory stamps a provisional deployment's model_info into kwargs before the prompt pass runs, so an injection recorded there named that provisional pick and a differently-billed deployment lost the credit - record_gateway_injection overwrote on every positive delta, so a per-leg stamp (the Bedrock converse tool_config one included) downgraded an existing every-deployment mark and the leg billed after a failover lost the credit record_gateway_injection now takes injected_for_every_deployment, the two pre-choice callers declare it, and an every-deployment mark is never narrowed by a later per-leg stamp. Per-leg marks still overwrite each other. Spend amounts are untouched; only the savings attribution is affected. Also unblocks make lint at the staging tip: tests/e2e/test_junit_properties.py landed three basedpyright reds via an e2e-only PR whose lint job skipped, now suppressed as the deliberate duck-typed double they are. --- .../anthropic_cache_control_hook.py | 32 +++++++++--- litellm/litellm_core_utils/litellm_logging.py | 4 ++ litellm/proxy/utils.py | 1 + litellm/router.py | 1 + tests/e2e/test_junit_properties.py | 6 +-- .../test_anthropic_cache_control_hook.py | 21 ++++++++ .../test_litellm_logging.py | 26 +++++++++- tests/test_litellm/test_router.py | 52 +++++++++++++++++++ 8 files changed, 131 insertions(+), 12 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 545b0f40018..3519240dda9 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -755,6 +755,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): def record_gateway_injection( request_kwargs: Mapping[str, object], added: int, + injected_for_every_deployment: bool = False, ) -> None: """Name the deployment whose payload the gateway, not the client, put breakpoints on. @@ -771,7 +772,16 @@ class AnthropicCacheControlHook(CustomPromptManagement): A pass that runs before a deployment is chosen, which is what the proxy does for prompt templates, injects into the payload every leg goes on to send, so it marks - the request for all of them rather than for one. + the request for all of them rather than for one. Such a pass says so with + ``injected_for_every_deployment`` instead of relying on the shape of + ``request_kwargs``: the router's prompt-management factory stamps a provisional + deployment's ``model_info`` into kwargs before the prompt pass runs, and billing + the request through any other deployment would silently drop the credit. An + every-deployment mark, once written, also never narrows: a later per-leg stamp + (the Bedrock converse tool_config one included) describes one leg of a payload + every leg sends, so narrowing to it would uncredit whichever leg gets billed + after a failover. Both losses are fail-closed under-crediting, which is why the + guard only protects the sentinel and per-leg marks still overwrite each other. Only what this pass actually placed counts. A ``tool_config`` point is placed by the Bedrock converse transform, and only when the request carries tools, so the @@ -801,13 +811,19 @@ class AnthropicCacheControlHook(CustomPromptManagement): ), None, ) - if bucket is not None: - model_info: Final = request_kwargs.get("model_info") - bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = ( - model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT) - if isinstance(model_info, dict) - else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT - ) + if bucket is None: + return + if bucket.get(GATEWAY_INJECTED_CACHE_METADATA_KEY) == GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT: + return + if injected_for_every_deployment: + bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT + return + model_info: Final = request_kwargs.get("model_info") + bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = ( + model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT) + if isinstance(model_info, dict) + else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT + ) @staticmethod def maybe_inject_cache_control( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9a6fb11f978..f94e86b4460 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -901,6 +901,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_label: str | None = None, prompt_version: int | None = None, request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs + injected_for_every_deployment: bool = False, ) -> tuple[str, list[AllMessageValues], dict]: from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook @@ -933,6 +934,7 @@ class Logging(LiteLLMLoggingBaseClass): AnthropicCacheControlHook.record_gateway_injection( request_kwargs, AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before, + injected_for_every_deployment=injected_for_every_deployment, ) self.messages = messages return model, messages, non_default_params @@ -950,6 +952,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_label: str | None = None, prompt_version: int | None = None, request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs + injected_for_every_deployment: bool = False, ) -> tuple[str, list[AllMessageValues], dict]: from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook @@ -985,6 +988,7 @@ class Logging(LiteLLMLoggingBaseClass): AnthropicCacheControlHook.record_gateway_injection( request_kwargs, AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before, + injected_for_every_deployment=injected_for_every_deployment, ) self.messages = messages return model, messages, non_default_params diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 051d36c4d0f..cab2bd6d9db 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1524,6 +1524,7 @@ class ProxyLogging: prompt_label=data.pop("prompt_label", None) or {}, prompt_version=data.pop("prompt_version", None) or {}, request_kwargs=data, + injected_for_every_deployment=True, ) data.update(optional_params) diff --git a/litellm/router.py b/litellm/router.py index 6e4405ebfef..462d5414456 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4006,6 +4006,7 @@ class Router: prompt_variables=prompt_variables, prompt_label=prompt_label, request_kwargs=kwargs, + injected_for_every_deployment=True, ) # Filter out prompt management specific parameters from data before merging diff --git a/tests/e2e/test_junit_properties.py b/tests/e2e/test_junit_properties.py index c0596177cc1..f7d1f70c5ec 100644 --- a/tests/e2e/test_junit_properties.py +++ b/tests/e2e/test_junit_properties.py @@ -115,7 +115,7 @@ class TestResultProperties: ("logging/test_x.py", 40, "TestFoo.test_bar"), (FakeMarker("covers", "LOG-1", "LOG-2"),), ) - assert result_properties(item) == ( + assert result_properties(item) == ( # pyright: ignore[reportArgumentType] # duck-typed Item double ("package", "logging"), ("covers", "LOG-1,LOG-2"), ("source", "tests/e2e/logging/test_x.py:41"), @@ -125,8 +125,8 @@ class TestResultProperties: """Collection can run the hook more than once; a second pass must not double the entries in the report.""" item = FakeItem("logging/test_x.py::test_bar", ("logging/test_x.py", 40, "test_bar")) - attach_result_properties(item) - attach_result_properties(item) + attach_result_properties(item) # pyright: ignore[reportArgumentType] # duck-typed Item double + attach_result_properties(item) # pyright: ignore[reportArgumentType] # duck-typed Item double assert [name for name, _ in item.user_properties] == ["package", "covers", "source"] diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index e995cbae782..de8b654987b 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -2858,6 +2858,27 @@ class TestRecordGatewayInjection: AnthropicCacheControlHook.record_gateway_injection(kwargs, 0) assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT + def test_an_every_deployment_mark_survives_a_later_per_deployment_stamp(self): + """A per-leg stamp like the Bedrock converse tool_config one describes one leg of + a payload every leg sends, so narrowing an every-deployment mark to that leg's + deployment would uncredit whichever leg gets billed after a failover.""" + kwargs: dict = {"litellm_metadata": {self.KEY: ""}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 1) + assert kwargs["litellm_metadata"][self.KEY] == "" + + def test_a_pre_choice_pass_stamps_the_sentinel_over_a_provisional_deployment(self): + """The router's prompt-management factory stamps a provisional deployment's + model_info into kwargs before the prompt pass runs, and any other deployment can + end up billed, so the pass declares every-deployment scope explicitly.""" + kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 1, injected_for_every_deployment=True) + assert kwargs["litellm_metadata"][self.KEY] == "" + + def test_a_per_deployment_mark_still_follows_the_latest_leg(self): + kwargs: dict = {"litellm_metadata": {self.KEY: "dep-old"}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 1) + assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT + def test_v1_messages_auto_injection_stamps_the_marker(self, monkeypatch): monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}} diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 366f61ded49..f1de7390b5b 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6002,7 +6002,9 @@ async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_o """The savings gate reads litellm_gateway_injected_cache from the request's metadata bucket. Recording lives in the shared prompt-hook wrappers, so chat, /v1/responses, router prompt deployments, and proxy prompt templates all mark - injected requests the same way; a hook that injects nothing leaves no marker.""" + injected requests the same way; a hook that injects nothing leaves no marker. + A pass that runs before deployment choice declares it and gets the every-deployment + sentinel, which a later per-deployment pass never narrows.""" from litellm.integrations.custom_prompt_management import CustomPromptManagement class _InjectingHook(CustomPromptManagement): @@ -6086,6 +6088,28 @@ async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_o ) assert "litellm_gateway_injected_cache" not in untouched["metadata"] + pre_choice = {"metadata": {}, "model_info": {"id": "dep-of-this-attempt"}} + logging_obj.get_chat_completion_prompt( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + non_default_params={}, + prompt_variables=None, + prompt_management_logger=_InjectingHook(), + request_kwargs=pre_choice, + injected_for_every_deployment=True, + ) + assert pre_choice["metadata"]["litellm_gateway_injected_cache"] == "" + + await logging_obj.async_get_chat_completion_prompt( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "a fresh turn"}], + non_default_params={}, + prompt_variables=None, + prompt_management_logger=_InjectingHook(), + request_kwargs=pre_choice, + ) + assert pre_choice["metadata"]["litellm_gateway_injected_cache"] == "" + def test_get_standard_logging_object_payload_reads_overhead_from_logging_obj_for_dict_results(logging_obj): """LIT-5466: /v1/messages returns a plain dict with no _hidden_params, so the overhead diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 84f6344be35..058ed5bb3a7 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -11730,3 +11730,55 @@ class TestPreRoutingTierDrivesFallbacks: response = await router.acompletion(model="smart-router", messages=[{"role": "user", "content": "hi"}]) assert response.choices[0].message.content == "from backup-b" + + +@pytest.mark.asyncio +async def test_prompt_management_factory_marks_injection_for_every_deployment(monkeypatch): + """The factory stamps a provisional deployment's model_info into kwargs before the + prompt pass runs, then routes on the returned model, so any deployment can end up + billed. An injection recorded there must carry the every-deployment sentinel, never + the provisional deployment's id, or a differently-billed deployment loses the credit.""" + import time + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + router = litellm.Router( + model_list=[ + { + "model_name": "cached-claude", + "litellm_params": { + "model": "anthropic_cache_control_hook/claude-sonnet-5", + "prompt_id": "cache-points", + }, + "model_info": {"id": "provisional-dep"}, + } + ] + ) + captured: dict = {} + + async def _capture_acompletion(**kwargs): + captured.update(kwargs) + return litellm.ModelResponse() + + monkeypatch.setattr(litellm, "acompletion", _capture_acompletion) + logging_obj = LiteLLMLogging( + model="cached-claude", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + start_time=time.time(), + litellm_call_id="lit-6445", + function_id="f", + ) + await router.acompletion( + model="cached-claude", + messages=[ + {"role": "system", "content": "a static system prompt"}, + {"role": "user", "content": "hi"}, + ], + cache_control_injection_points=[{"location": "message", "role": "system"}], + litellm_logging_obj=logging_obj, + ) + bucket = captured.get("litellm_metadata") or captured["metadata"] + assert captured["model_info"]["id"] == "provisional-dep" + assert bucket["litellm_gateway_injected_cache"] == "" From e3a61c82da9f8dbe42fdfcc4907af7b3a2901392 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 1 Sep 2026 17:46:41 -0700 Subject: [PATCH 044/175] test(router): register indirect session routing coverage --- tests/code_coverage_tests/router_code_coverage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index a5e00799519..60b56b7fac6 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -82,6 +82,10 @@ ignored_function_names = [ "_invalidate_access_groups_cache", # Tested indirectly via set_model_list, upsert_model etc. (test files lack "router" in name) "has_buffered_provider_output", # Property, so its reads in test_router.py are never an ast.Call "_resolved_provider", # Tested via get_pattern in test_pattern_match_deployments.py (file lacks "router" in name) + "_request_header", # Tested through Claude Code session routing in test_router.py + "_claude_code_session_router_cache_key", # Tested through Claude Code session routing in test_router.py + "_delete_claude_code_session_router_binding", # Tested through Redis cleanup failure in test_router.py + "_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py ] From 2d4301589c1e741489f68e6eef3c2d112da91e2a Mon Sep 17 00:00:00 2001 From: Moe Khalil Date: Wed, 2 Sep 2026 01:10:31 +0000 Subject: [PATCH 045/175] fix(router): keep serving when Claude Code session router cleanup fails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 12 ++++-------- tests/test_litellm/test_router.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 2bb725b0880..d6d9f20085f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12578,15 +12578,11 @@ class Router: return None return f"claude_code_session_router:v1:{caller_scope}:{session_id}" - async def _delete_claude_code_session_router_binding(self, cache_key: str) -> None: + async def _clear_claude_code_session_router(self, cache_key: str) -> None: try: await self.cache.async_delete_cache(key=cache_key) except Exception as e: # noqa: BLE001 # cache cleanup must not fail an otherwise routable request - verbose_router_logger.warning( - "Failed to delete Claude Code session router binding; " - "the binding may remain until its TTL expires: %s", - e, - ) + verbose_router_logger.debug("Claude Code session router cleanup skipped for %s: %s", cache_key, e) async def _resolve_claude_code_session_router( self, @@ -12605,7 +12601,7 @@ class Router: return registered_model_name bound_registered_model: Final = self._get_model_from_alias(model=bound_model) or bound_model if self._select_pre_routing_strategy(bound_registered_model, request_kwargs) is None: - await self._delete_claude_code_session_router_binding(cache_key) + await self._clear_claude_code_session_router(cache_key) return registered_model_name await self.cache.async_set_cache( key=cache_key, @@ -12620,7 +12616,7 @@ class Router: if request_kwargs.get("fallback_depth") not in (None, 0): return registered_model_name if self._select_pre_routing_strategy(registered_model_name, request_kwargs) is None: - await self._delete_claude_code_session_router_binding(cache_key) + await self._clear_claude_code_session_router(cache_key) return registered_model_name await self.cache.async_set_cache( key=cache_key, diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 77b73a2d12a..b66dbf6aa7d 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8428,6 +8428,24 @@ class TestClaudeCodeSubagentSessionRouterBinding: assert response is None redis_cache.async_delete_cache.assert_awaited_once() + @pytest.mark.asyncio + async def test_main_direct_model_still_served_when_cache_delete_fails(self): + router = self._router() + await router.acompletion( + model="smart-router", messages=[{"role": "user", "content": "main turn"}], **self._request_kwargs() + ) + + async def failing_delete(key: str) -> None: + raise Exception("Redis circuit breaker is open — skipping async_delete_cache") + + router.cache.async_delete_cache = failing_delete + + response = await router.acompletion( + model="expensive-model", messages=[{"role": "user", "content": "direct turn"}], **self._request_kwargs() + ) + + assert response.choices[0].message.content == "expensive response" + @pytest.mark.asyncio async def test_background_and_fallback_requests_do_not_clear_the_session_router(self): router = self._router() From 46502f58042a41619648be7acc67108079f737e2 Mon Sep 17 00:00:00 2001 From: Moe Khalil Date: Wed, 2 Sep 2026 01:11:23 +0000 Subject: [PATCH 046/175] Revert "fix(router): keep serving when Claude Code session router cleanup fails" This reverts commit 2d4301589c1e741489f68e6eef3c2d112da91e2a. --- litellm/router.py | 12 ++++++++---- tests/test_litellm/test_router.py | 18 ------------------ 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d6d9f20085f..2bb725b0880 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12578,11 +12578,15 @@ class Router: return None return f"claude_code_session_router:v1:{caller_scope}:{session_id}" - async def _clear_claude_code_session_router(self, cache_key: str) -> None: + async def _delete_claude_code_session_router_binding(self, cache_key: str) -> None: try: await self.cache.async_delete_cache(key=cache_key) except Exception as e: # noqa: BLE001 # cache cleanup must not fail an otherwise routable request - verbose_router_logger.debug("Claude Code session router cleanup skipped for %s: %s", cache_key, e) + verbose_router_logger.warning( + "Failed to delete Claude Code session router binding; " + "the binding may remain until its TTL expires: %s", + e, + ) async def _resolve_claude_code_session_router( self, @@ -12601,7 +12605,7 @@ class Router: return registered_model_name bound_registered_model: Final = self._get_model_from_alias(model=bound_model) or bound_model if self._select_pre_routing_strategy(bound_registered_model, request_kwargs) is None: - await self._clear_claude_code_session_router(cache_key) + await self._delete_claude_code_session_router_binding(cache_key) return registered_model_name await self.cache.async_set_cache( key=cache_key, @@ -12616,7 +12620,7 @@ class Router: if request_kwargs.get("fallback_depth") not in (None, 0): return registered_model_name if self._select_pre_routing_strategy(registered_model_name, request_kwargs) is None: - await self._clear_claude_code_session_router(cache_key) + await self._delete_claude_code_session_router_binding(cache_key) return registered_model_name await self.cache.async_set_cache( key=cache_key, diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index b66dbf6aa7d..77b73a2d12a 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8428,24 +8428,6 @@ class TestClaudeCodeSubagentSessionRouterBinding: assert response is None redis_cache.async_delete_cache.assert_awaited_once() - @pytest.mark.asyncio - async def test_main_direct_model_still_served_when_cache_delete_fails(self): - router = self._router() - await router.acompletion( - model="smart-router", messages=[{"role": "user", "content": "main turn"}], **self._request_kwargs() - ) - - async def failing_delete(key: str) -> None: - raise Exception("Redis circuit breaker is open — skipping async_delete_cache") - - router.cache.async_delete_cache = failing_delete - - response = await router.acompletion( - model="expensive-model", messages=[{"role": "user", "content": "direct turn"}], **self._request_kwargs() - ) - - assert response.choices[0].message.content == "expensive response" - @pytest.mark.asyncio async def test_background_and_fallback_requests_do_not_clear_the_session_router(self): router = self._router() From f49a3e15a8b936c94e5d050017f6887e8d34e997 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:12:07 -0700 Subject: [PATCH 047/175] test(e2e): read JUnit properties off the real collected pytest Item tests/e2e/test_junit_properties.py fed a hand-rolled FakeItem to result_properties and attach_result_properties, both typed pytest.Item, so uv run basedpyright tests/e2e reported 3 reportArgumentType errors on litellm_internal_staging and every make check that scopes a litellm/ or tests/e2e/ Python file failed. Each test now looks up its own collected Item in request.session.items and applies the covers marker at run time through request.applymarker, so the coverage registry's collect-only pass never sees the test ids and the production functions keep their pytest.Item signatures. No casts, no ignores. Resolves LIT-6669 --- tests/e2e/test_junit_properties.py | 45 ++++++++++-------------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/tests/e2e/test_junit_properties.py b/tests/e2e/test_junit_properties.py index c0596177cc1..02c1413c840 100644 --- a/tests/e2e/test_junit_properties.py +++ b/tests/e2e/test_junit_properties.py @@ -24,25 +24,10 @@ from junit_properties import ( ) -class FakeMarker: - def __init__(self, name: str, *args: object) -> None: - self.name = name - self.args = args - - -class FakeItem: - """The three attributes junit_properties reads off a pytest Item.""" - - def __init__( - self, nodeid: str, location: tuple[str, int | None, str], markers: tuple[FakeMarker, ...] = () - ) -> None: - self.nodeid = nodeid - self.location = location - self.user_properties: list[tuple[str, str]] = [] - self._markers = markers - - def iter_markers(self, name: str): - return (marker for marker in self._markers if marker.name == name) +def collected_item(request: pytest.FixtureRequest, name: str) -> pytest.Item: + """The Item pytest collected for test ``name`` in this file: the real nodeid, + location and marker machinery the collection hook reads, as pytest built it.""" + return next(item for item in request.session.items if item.path == request.path and item.name == name) def repo_root() -> Path | None: @@ -109,22 +94,22 @@ class TestSourceFromLocation: class TestResultProperties: - def test_every_test_carries_package_covers_and_source(self) -> None: - item = FakeItem( - "logging/test_x.py::TestFoo::test_bar", - ("logging/test_x.py", 40, "TestFoo.test_bar"), - (FakeMarker("covers", "LOG-1", "LOG-2"),), - ) - assert result_properties(item) == ( - ("package", "logging"), + def test_every_test_carries_package_covers_and_source(self, request: pytest.FixtureRequest) -> None: + """Read off this test's own collected Item, so the nodeid and location are + whatever pytest reports for the launch shape in use, and the marker is added + at run time so the coverage registry's collect-only pass never sees it.""" + test = type(self).test_every_test_carries_package_covers_and_source + request.applymarker(pytest.mark.covers("LOG-1", "LOG-2")) + assert result_properties(collected_item(request, test.__name__)) == ( + ("package", "root"), ("covers", "LOG-1,LOG-2"), - ("source", "tests/e2e/logging/test_x.py:41"), + ("source", f"tests/e2e/test_junit_properties.py:{test.__code__.co_firstlineno}"), ) - def test_attach_is_idempotent(self) -> None: + def test_attach_is_idempotent(self, request: pytest.FixtureRequest) -> None: """Collection can run the hook more than once; a second pass must not double the entries in the report.""" - item = FakeItem("logging/test_x.py::test_bar", ("logging/test_x.py", 40, "test_bar")) + item = collected_item(request, type(self).test_attach_is_idempotent.__name__) attach_result_properties(item) attach_result_properties(item) assert [name for name, _ in item.user_properties] == ["package", "covers", "source"] From 0608f0a00f2c76b50640ed7f4559f4c8551fdc44 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:17:02 +0000 Subject: [PATCH 048/175] fix: reject unknown runtime router settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 22 +++++++++++ litellm/proxy/proxy_server.py | 21 +++++++++- litellm/router.py | 30 ++++---------- litellm/types/router.py | 29 +++++++------- .../proxy/proxy_server/test_routes_config.py | 39 +++++++++++++++++++ .../test_router_retry_policy_update.py | 21 +++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 7 files changed, 125 insertions(+), 39 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 1bd977dd9a9..a7506cb6378 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -38,6 +38,28 @@ DEFAULT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_MAX_TOKENS", 4096)) DEFAULT_ALLOWED_FAILS: Final = int(os.getenv("DEFAULT_ALLOWED_FAILS", 3)) DEFAULT_REDIS_SYNC_INTERVAL: Final = int(os.getenv("DEFAULT_REDIS_SYNC_INTERVAL", 1)) DEFAULT_COOLDOWN_TIME_SECONDS: Final = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECONDS", 5)) +RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( + { + "routing_strategy_args", + "routing_strategy", + "routing_groups", + "allowed_fails", + "cooldown_time", + "num_retries", + "timeout", + "max_retries", + "retry_after", + "fallbacks", + "context_window_fallbacks", + "retry_policy", + "model_group_retry_policy", + "model_group_alias", + "enable_weighted_failover", + "enable_tag_filtering", + "tag_routing_prefix", + "optional_pre_call_checks", + } +) DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5)) DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)) DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 77a80ea0052..9de6b38265a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -39,7 +39,7 @@ from typing import ( import anyio import websockets import websockets.exceptions -from pydantic import BaseModel, Json, JsonValue, ValidationError +from pydantic import BaseModel, Json, JsonValue, TypeAdapter, ValidationError from typing_extensions import NotRequired, ReadOnly, assert_never from litellm._uuid import uuid @@ -60,6 +60,7 @@ from litellm.constants import ( LITELLM_SETTINGS_SAFE_DB_OVERRIDES, LITELLM_UI_ALLOW_HEADERS, LITELLM_UI_SESSION_DURATION, + RUNTIME_UPDATABLE_ROUTER_SETTINGS, ) from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, @@ -16207,6 +16208,7 @@ async def invitation_delete( ) async def update_config( config_info: ConfigYAML, + request: Request, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -16218,6 +16220,23 @@ async def update_config( a side effect of an unrelated update. """ global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client + request_body: Final[Mapping[str, JsonValue]] = TypeAdapter(Mapping[str, JsonValue]).validate_python( + await request.json() + ) + raw_router_settings: Final = request_body.get("router_settings") + if isinstance(raw_router_settings, dict): + unsupported_router_settings: Final = sorted(set(raw_router_settings) - RUNTIME_UPDATABLE_ROUTER_SETTINGS) + if unsupported_router_settings: + raise HTTPException( + status_code=400, + detail={ + "error": ( + f"Unsupported router settings: {', '.join(unsupported_router_settings)} " + "are not runtime-updatable router settings" + ) + }, + ) + try: if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException(status_code=403, detail="Only proxy admins can update config") diff --git a/litellm/router.py b/litellm/router.py index 23d8907fb49..fc4f815170d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -50,6 +50,7 @@ from litellm.constants import ( DEFAULT_HEALTH_CHECK_INTERVAL, DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER, DEFAULT_MAX_LRU_CACHE_SIZE, + RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger @@ -2072,6 +2073,10 @@ class Router: if _callback is None: continue + if self.optional_callbacks is not None and any( + isinstance(callback, type(_callback)) for callback in self.optional_callbacks + ): + continue if self.optional_callbacks is None: self.optional_callbacks = [] self.optional_callbacks.append(_callback) @@ -11331,27 +11336,6 @@ class Router: """ Update the router settings. """ - # only the following settings are allowed to be configured - _allowed_settings: Final = [ - "routing_strategy_args", - "routing_strategy", - "routing_groups", - "allowed_fails", - "cooldown_time", - "num_retries", - "timeout", - "max_retries", - "retry_after", - "fallbacks", - "context_window_fallbacks", - "retry_policy", - "model_group_retry_policy", - "model_group_alias", - "enable_weighted_failover", - "enable_tag_filtering", - "tag_routing_prefix", - ] - _int_settings: Final = [ "timeout", "num_retries", @@ -11364,13 +11348,15 @@ class Router: rebuild_routing_groups = False relink_lar1_from_args = False for var in kwargs: - if var in _allowed_settings: + if var in RUNTIME_UPDATABLE_ROUTER_SETTINGS: if var in _int_settings: _casted_value = int(kwargs[var]) setattr(self, var, _casted_value) elif var == "routing_groups": self._routing_groups_input = kwargs[var] rebuild_routing_groups = True + elif var == "optional_pre_call_checks": + self.add_optional_pre_call_checks(kwargs[var]) elif var == "retry_policy": value = kwargs[var] if isinstance(value, dict): diff --git a/litellm/types/router.py b/litellm/types/router.py index e0957383aac..2a5f264cee3 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -106,6 +106,20 @@ class RetryPolicy(BaseModel): InternalServerErrorRetries: int | None = None +OptionalPreCallChecks = list[ + Literal[ + "prompt_caching", + "router_budget_limiting", + "responses_api_deployment_check", + "deployment_affinity", + "session_affinity", + "forward_client_headers_by_model_group", + "enforce_model_rate_limits", + "encrypted_content_affinity", + ] +] + + class UpdateRouterConfig(BaseModel): """ Set of params that you can modify via `router.update_settings()`. @@ -128,6 +142,7 @@ class UpdateRouterConfig(BaseModel): model_group_alias: dict[str, str | dict] | None = {} enable_tag_filtering: bool | None = None tag_routing_prefix: str | None = None + optional_pre_call_checks: OptionalPreCallChecks | None = None model_config = ConfigDict(protected_namespaces=()) @@ -869,20 +884,6 @@ class FallbackAccessCheck(Protocol): async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: "Router") -> bool: ... -OptionalPreCallChecks = list[ - Literal[ - "prompt_caching", - "router_budget_limiting", - "responses_api_deployment_check", - "deployment_affinity", - "session_affinity", - "forward_client_headers_by_model_group", - "enforce_model_rate_limits", - "encrypted_content_affinity", - ] -] - - class LiteLLM_RouterFileObject(TypedDict, total=False): """ Tracking the litellm params hash, used for mapping the file id to the right model diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index ad3c470acf3..0df8fb663e2 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -60,6 +60,45 @@ def test_config_update_happy_admin(client, auth_as, mock_prisma, monkeypatch): assert normalize(response.json()) == {"message": "Config updated successfully"} +def test_config_update_persists_optional_pre_call_checks(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + fake_proxy_config = MagicMock() + fake_proxy_config.add_deployment = AsyncMock() + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"optional_pre_call_checks": ["prompt_caching"]}}, + ) + + assert response.status_code == 200 + persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"]) + assert persisted["optional_pre_call_checks"] == ["prompt_caching"] + + +def test_config_update_rejects_unknown_router_setting(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"optional_precall_checks": ["prompt_caching"]}}, + ) + + assert response.status_code == 400 + assert "optional_precall_checks" in response.json()["detail"]["error"] + table.upsert.assert_not_called() + + def test_config_update_non_admin_forbidden(client, auth_as, mock_prisma, monkeypatch): """POST /config/update by a non-admin caller is rejected; the error surfaces as a ProxyException with the admin-only message.""" diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 1b98b8c1ae8..1b014cd8401 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -26,8 +26,8 @@ from unittest.mock import AsyncMock, MagicMock import pytest from pydantic import ValidationError - import litellm +from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck from litellm.types.router import RetryPolicy, UpdateRouterConfig # --------------------------------------------------------------------------- @@ -100,6 +100,19 @@ def _build_router() -> litellm.Router: ) +def test_update_settings_adds_optional_pre_call_check_once(): + router = _build_router() + + router.update_settings(num_retries=7, optional_pre_call_checks=["prompt_caching"]) + router.update_settings(optional_pre_call_checks=["prompt_caching"]) + + prompt_caching_callbacks = [ + callback for callback in router.optional_callbacks if isinstance(callback, PromptCachingDeploymentCheck) + ] + assert len(prompt_caching_callbacks) == 1 + assert router.num_retries == 7 + + def test_update_settings_persists_retry_policy_dict(): """When the proxy's ``_add_router_settings_from_db_config`` calls ``llm_router.update_settings(retry_policy={...})`` after reading the @@ -228,7 +241,7 @@ async def test_config_update_persists_and_reads_back_retry_policy(monkeypatch): """The exact global retry_policy save the UI performs must survive the real ``/config/update`` -> DB -> apply -> ``/get/config/callbacks`` path, not snap back to the ``num_retries`` fallback the ticket reported.""" - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import ConfigYAML, LitellmUserRoles, UserAPIKeyAuth router = _build_router() @@ -255,8 +268,12 @@ async def test_config_update_persists_and_reads_back_retry_policy(monkeypatch): RateLimitErrorRetries=7, ) ) + request = MagicMock() + request.json = AsyncMock(return_value={"router_settings": {"retry_policy": posted.model_dump()}}) + await proxy_server.update_config( config_info=ConfigYAML(router_settings=posted), + request=request, user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234"), ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6f044fec3f3..bde7fd611d5 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -37473,6 +37473,8 @@ export interface components { } | null; /** Num Retries */ num_retries?: number | null; + /** Optional Pre Call Checks */ + optional_pre_call_checks?: ("prompt_caching" | "router_budget_limiting" | "responses_api_deployment_check" | "deployment_affinity" | "session_affinity" | "forward_client_headers_by_model_group" | "enforce_model_rate_limits" | "encrypted_content_affinity")[] | null; /** Retry After */ retry_after?: number | null; retry_policy?: components["schemas"]["RetryPolicy"] | null; From cb511f70ccbee8cc9257b52cc6ad5d7721219ce3 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:20:51 +0000 Subject: [PATCH 049/175] fix: preserve config update authorization order Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 44 +++++++++---------- litellm/proxy/proxy_server.py | 34 +++++++------- .../proxy/proxy_server/test_routes_config.py | 19 +++++++- .../test_router_retry_policy_update.py | 3 +- 4 files changed, 59 insertions(+), 41 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index a7506cb6378..5b44e8b5f51 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -9,6 +9,28 @@ DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT" AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5)) ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000 +RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( + { + "routing_strategy_args", + "routing_strategy", + "routing_groups", + "allowed_fails", + "cooldown_time", + "num_retries", + "timeout", + "max_retries", + "retry_after", + "fallbacks", + "context_window_fallbacks", + "retry_policy", + "model_group_retry_policy", + "model_group_alias", + "enable_weighted_failover", + "enable_tag_filtering", + "tag_routing_prefix", + "optional_pre_call_checks", + } +) DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) @@ -38,28 +60,6 @@ DEFAULT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_MAX_TOKENS", 4096)) DEFAULT_ALLOWED_FAILS: Final = int(os.getenv("DEFAULT_ALLOWED_FAILS", 3)) DEFAULT_REDIS_SYNC_INTERVAL: Final = int(os.getenv("DEFAULT_REDIS_SYNC_INTERVAL", 1)) DEFAULT_COOLDOWN_TIME_SECONDS: Final = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECONDS", 5)) -RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( - { - "routing_strategy_args", - "routing_strategy", - "routing_groups", - "allowed_fails", - "cooldown_time", - "num_retries", - "timeout", - "max_retries", - "retry_after", - "fallbacks", - "context_window_fallbacks", - "retry_policy", - "model_group_retry_policy", - "model_group_alias", - "enable_weighted_failover", - "enable_tag_filtering", - "tag_routing_prefix", - "optional_pre_call_checks", - } -) DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5)) DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)) DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9de6b38265a..dbeb8486539 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16220,27 +16220,27 @@ async def update_config( a side effect of an unrelated update. """ global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client - request_body: Final[Mapping[str, JsonValue]] = TypeAdapter(Mapping[str, JsonValue]).validate_python( - await request.json() - ) - raw_router_settings: Final = request_body.get("router_settings") - if isinstance(raw_router_settings, dict): - unsupported_router_settings: Final = sorted(set(raw_router_settings) - RUNTIME_UPDATABLE_ROUTER_SETTINGS) - if unsupported_router_settings: - raise HTTPException( - status_code=400, - detail={ - "error": ( - f"Unsupported router settings: {', '.join(unsupported_router_settings)} " - "are not runtime-updatable router settings" - ) - }, - ) - try: if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException(status_code=403, detail="Only proxy admins can update config") + request_body: Final[Mapping[str, JsonValue]] = TypeAdapter(Mapping[str, JsonValue]).validate_python( + await request.json() + ) + raw_router_settings: Final = request_body.get("router_settings") + if isinstance(raw_router_settings, dict): + unsupported_router_settings: Final = sorted(set(raw_router_settings) - RUNTIME_UPDATABLE_ROUTER_SETTINGS) + if unsupported_router_settings: + raise HTTPException( + status_code=400, + detail={ + "error": ( + f"Unsupported router settings: {', '.join(unsupported_router_settings)} " + "are not runtime-updatable router settings" + ) + }, + ) + if prisma_client is None: raise Exception("No DB Connected") diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index 0df8fb663e2..4b0954c350a 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -95,10 +95,27 @@ def test_config_update_rejects_unknown_router_setting(client, auth_as, mock_pris ) assert response.status_code == 400 - assert "optional_precall_checks" in response.json()["detail"]["error"] + assert "optional_precall_checks" in response.json()["error"]["message"] table.upsert.assert_not_called() +def test_config_update_unknown_router_setting_non_admin_forbidden(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.INTERNAL_USER): + response = client.post( + "/config/update", + json={"router_settings": {"optional_precall_checks": ["prompt_caching"]}}, + ) + + assert response.status_code == 403 + assert "admin" in response.json()["error"]["message"].lower() + + def test_config_update_non_admin_forbidden(client, auth_as, mock_prisma, monkeypatch): """POST /config/update by a non-admin caller is rejected; the error surfaces as a ProxyException with the admin-only message.""" diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 1b014cd8401..e386eebf3d9 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -26,6 +26,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from pydantic import ValidationError + import litellm from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck from litellm.types.router import RetryPolicy, UpdateRouterConfig @@ -241,7 +242,7 @@ async def test_config_update_persists_and_reads_back_retry_policy(monkeypatch): """The exact global retry_policy save the UI performs must survive the real ``/config/update`` -> DB -> apply -> ``/get/config/callbacks`` path, not snap back to the ``num_retries`` fallback the ticket reported.""" - from litellm.proxy import proxy_server + import litellm.proxy.proxy_server as proxy_server from litellm.proxy._types import ConfigYAML, LitellmUserRoles, UserAPIKeyAuth router = _build_router() From 29f0110fe08d5b8f4798e20eed6d2368e4cada2d Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:29:41 +0000 Subject: [PATCH 050/175] test: pass request to config update test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/proxy_unit_tests/test_proxy_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 47554913419..54cce9cdd78 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -3076,7 +3076,9 @@ async def test_update_config_success_callback_normalization(): admin_user = UserAPIKeyAuth( user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test" ) - await proxy_server.update_config(config_update, user_api_key_dict=admin_user) + request = MagicMock() + request.json = AsyncMock(return_value={"litellm_settings": {"success_callback": ["SQS", "sQs"]}}) + await proxy_server.update_config(config_update, request=request, user_api_key_dict=admin_user) assert ( "litellm_settings" in upserted From e67f98feb1cb1758e253beaf005eaaad44ab3abe Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:44:17 +0000 Subject: [PATCH 051/175] fix: reconcile runtime pre-call checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 30 +++++++++++++- .../test_router_retry_policy_update.py | 40 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index fc4f815170d..e2865542e89 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -355,6 +355,13 @@ _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" +_RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS: Final[Mapping[str, type[CustomLogger]]] = MappingProxyType( + { + "prompt_caching": PromptCachingDeploymentCheck, + "enforce_model_rate_limits": ModelRateLimitingCheck, + } +) + def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool: for chunk in chunks: @@ -2082,6 +2089,27 @@ class Router: self.optional_callbacks.append(_callback) litellm.logging_callback_manager.add_litellm_callback(_callback) + def set_optional_pre_call_checks(self, optional_pre_call_checks: OptionalPreCallChecks | None) -> None: + if optional_pre_call_checks is None: + return + requested: Final = frozenset(optional_pre_call_checks) + for name, callback_cls in _RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS.items(): + if name not in requested: + self._remove_optional_callbacks_of_type(callback_cls) + self.add_optional_pre_call_checks(optional_pre_call_checks) + + def _remove_optional_callbacks_of_type(self, callback_cls: type[CustomLogger]) -> None: + if self.optional_callbacks is None: + return + removed: Final = [cb for cb in self.optional_callbacks if isinstance(cb, callback_cls)] + if not removed: + return + self.optional_callbacks = [cb for cb in self.optional_callbacks if not isinstance(cb, callback_cls)] + for cb in removed: + litellm.logging_callback_manager.remove_callback_from_list_by_object( + litellm.callbacks, cb, require_self=False + ) + def print_deployment(self, deployment: dict): """ returns a copy of the deployment with the api key masked @@ -11356,7 +11384,7 @@ class Router: self._routing_groups_input = kwargs[var] rebuild_routing_groups = True elif var == "optional_pre_call_checks": - self.add_optional_pre_call_checks(kwargs[var]) + self.set_optional_pre_call_checks(kwargs[var]) elif var == "retry_policy": value = kwargs[var] if isinstance(value, dict): diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index e386eebf3d9..2c23d0da7e7 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -28,6 +28,8 @@ from pydantic import ValidationError import litellm +from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck from litellm.types.router import RetryPolicy, UpdateRouterConfig @@ -114,6 +116,44 @@ def test_update_settings_adds_optional_pre_call_check_once(): assert router.num_retries == 7 +def test_update_settings_clears_omitted_toggleable_pre_call_checks(): + router = _build_router() + + router.update_settings(optional_pre_call_checks=["prompt_caching"]) + router.update_settings(optional_pre_call_checks=[]) + + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + + +def test_update_settings_replaces_toggleable_pre_call_checks(): + router = _build_router() + + router.update_settings(optional_pre_call_checks=["prompt_caching"]) + router.update_settings(optional_pre_call_checks=["enforce_model_rate_limits"]) + + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + assert any(isinstance(callback, ModelRateLimitingCheck) for callback in (router.optional_callbacks or [])) + + +@pytest.mark.asyncio +async def test_update_settings_preserves_router_budget_limiting_when_omitted(monkeypatch): + async def _disable_periodic_sync(*args, **kwargs): + return None + + monkeypatch.setattr( + "litellm.router_strategy.budget_limiter.RouterBudgetLimiting.periodic_sync_in_memory_spend_with_redis", + _disable_periodic_sync, + ) + router = _build_router() + + router.add_optional_pre_call_checks(["router_budget_limiting"]) + router.update_settings(optional_pre_call_checks=[]) + + assert any(isinstance(callback, RouterBudgetLimiting) for callback in (router.optional_callbacks or [])) + + def test_update_settings_persists_retry_policy_dict(): """When the proxy's ``_add_router_settings_from_db_config`` calls ``llm_router.update_settings(retry_policy={...})`` after reading the From 7b86b7f4cd7576aeba56d9721f28002a4e5c6383 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:45:45 +0000 Subject: [PATCH 052/175] test: isolate router callback state Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_router_retry_policy_update.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 2c23d0da7e7..db710f76887 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -21,6 +21,7 @@ This file pins both halves of the fix. import json from dataclasses import dataclass +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -33,6 +34,14 @@ from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRat from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck from litellm.types.router import RetryPolicy, UpdateRouterConfig + +@pytest.fixture(autouse=True) +def isolate_litellm_callbacks(): + callbacks_before: Final = litellm.callbacks.copy() + yield + litellm.callbacks = callbacks_before + + # --------------------------------------------------------------------------- # UpdateRouterConfig schema membership (LIT-3152 part 1) # --------------------------------------------------------------------------- From 3dea586d3be34c52156fe764051dd8d750d4570b Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:48:09 +0000 Subject: [PATCH 053/175] test: cover runtime callback reconciliation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_router_retry_policy_update.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index db710f76887..9e0bb0b9bef 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -135,6 +135,26 @@ def test_update_settings_clears_omitted_toggleable_pre_call_checks(): assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) +def test_set_optional_pre_call_checks_reconciles_callback_types(): + router = _build_router() + + router.set_optional_pre_call_checks(["prompt_caching"]) + router.set_optional_pre_call_checks([]) + + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + + +def test_remove_optional_pre_call_check_removes_local_and_global_callbacks(): + router = _build_router() + + router.set_optional_pre_call_checks(["prompt_caching"]) + router._remove_optional_callbacks_of_type(PromptCachingDeploymentCheck) + + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + + def test_update_settings_replaces_toggleable_pre_call_checks(): router = _build_router() From 70a4f74a0d65bd89a9e9127076efd4301b987a29 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:56:23 +0000 Subject: [PATCH 054/175] test: allow callback state fixture mutation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_router_retry_policy_update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 9e0bb0b9bef..26251290176 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -39,7 +39,7 @@ from litellm.types.router import RetryPolicy, UpdateRouterConfig def isolate_litellm_callbacks(): callbacks_before: Final = litellm.callbacks.copy() yield - litellm.callbacks = callbacks_before + litellm.callbacks = callbacks_before # test-quality-ok: required callback-state restoration fixture # --------------------------------------------------------------------------- From 1d3e26fd98b40d40f498e14b2470e8acc79fb9f6 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 03:28:04 +0000 Subject: [PATCH 055/175] fix: preserve shared optional callbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 19 +++++---- .../test_router_retry_policy_update.py | 41 ++++++++++++++++++- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index e2865542e89..e7588d8ad5a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2099,16 +2099,19 @@ class Router: self.add_optional_pre_call_checks(optional_pre_call_checks) def _remove_optional_callbacks_of_type(self, callback_cls: type[CustomLogger]) -> None: - if self.optional_callbacks is None: + if self.optional_callbacks is None or not any(type(cb) is callback_cls for cb in self.optional_callbacks): return - removed: Final = [cb for cb in self.optional_callbacks if isinstance(cb, callback_cls)] - if not removed: + self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls] + if any( + router is not self and any(type(cb) is callback_cls for cb in (router.optional_callbacks or [])) + for router in tuple(_live_routers) + ): return - self.optional_callbacks = [cb for cb in self.optional_callbacks if not isinstance(cb, callback_cls)] - for cb in removed: - litellm.logging_callback_manager.remove_callback_from_list_by_object( - litellm.callbacks, cb, require_self=False - ) + for cb in tuple(litellm.callbacks): + if type(cb) is callback_cls: + litellm.logging_callback_manager.remove_callback_from_list_by_object( + litellm.callbacks, cb, require_self=False + ) def print_deployment(self, deployment: dict): """ diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 26251290176..be568134763 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -151,8 +151,45 @@ def test_remove_optional_pre_call_check_removes_local_and_global_callbacks(): router.set_optional_pre_call_checks(["prompt_caching"]) router._remove_optional_callbacks_of_type(PromptCachingDeploymentCheck) - assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) - assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router.optional_callbacks or [])) + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + +def test_remove_optional_pre_call_check_keeps_global_callback_for_another_router(): + router_a = _build_router() + router_b = _build_router() + + router_a.update_settings(optional_pre_call_checks=["prompt_caching"]) + router_b.update_settings(optional_pre_call_checks=["prompt_caching"]) + + router_a.update_settings(optional_pre_call_checks=[]) + + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router_a.optional_callbacks or [])) + assert any(type(callback) is PromptCachingDeploymentCheck for callback in (router_b.optional_callbacks or [])) + assert any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + router_b.update_settings(optional_pre_call_checks=[]) + + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router_b.optional_callbacks or [])) + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + +def test_remove_optional_pre_call_check_keeps_global_callback_when_second_router_clears_first(): + router_a = _build_router() + router_b = _build_router() + + router_a.update_settings(optional_pre_call_checks=["prompt_caching"]) + router_b.update_settings(optional_pre_call_checks=["prompt_caching"]) + + router_b.update_settings(optional_pre_call_checks=[]) + + assert any(type(callback) is PromptCachingDeploymentCheck for callback in (router_a.optional_callbacks or [])) + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router_b.optional_callbacks or [])) + assert any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + router_a.update_settings(optional_pre_call_checks=[]) + + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) def test_update_settings_replaces_toggleable_pre_call_checks(): From 8a0967443d84ecc02c10caf6ca55385b907b11b2 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 1 Sep 2026 20:38:13 -0700 Subject: [PATCH 056/175] fix(router): isolate Claude session binding cache --- litellm/router.py | 19 +++++++++++++------ tests/test_litellm/test_router.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 2bb725b0880..fb8625bc6ba 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -782,6 +782,10 @@ class Router: self.cache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache() ) # use a dual cache (Redis+In-Memory) for tracking cooldowns, usage, etc. + self._claude_code_session_router_cache: DualCache = DualCache( + redis_cache=redis_cache, + in_memory_cache=InMemoryCache(), + ) ### SCHEDULER ### self.scheduler = Scheduler(polling_interval=polling_interval, redis_cache=redis_cache) @@ -1102,8 +1106,8 @@ class Router: ``` and caching to just work. """ - if self.cache.redis_cache is None: - self.cache.redis_cache = cache + self.cache.attach_redis_cache(cache) + self._claude_code_session_router_cache.attach_redis_cache(cache) # Maps a routing strategy string to the attribute on `self` that holds # the default group's strategy selector for that strategy. (The selectors @@ -12580,7 +12584,7 @@ class Router: async def _delete_claude_code_session_router_binding(self, cache_key: str) -> None: try: - await self.cache.async_delete_cache(key=cache_key) + await self._claude_code_session_router_cache.async_delete_cache(key=cache_key) except Exception as e: # noqa: BLE001 # cache cleanup must not fail an otherwise routable request verbose_router_logger.warning( "Failed to delete Claude Code session router binding; " @@ -12600,14 +12604,14 @@ class Router: agent_id: Final = self._request_header(request_kwargs, "x-claude-code-agent-id") if agent_id is not None: - bound_model: Final = await self.cache.async_get_cache(key=cache_key) + bound_model: Final = await self._claude_code_session_router_cache.async_get_cache(key=cache_key) if not isinstance(bound_model, str): return registered_model_name bound_registered_model: Final = self._get_model_from_alias(model=bound_model) or bound_model if self._select_pre_routing_strategy(bound_registered_model, request_kwargs) is None: await self._delete_claude_code_session_router_binding(cache_key) return registered_model_name - await self.cache.async_set_cache( + await self._claude_code_session_router_cache.async_set_cache( key=cache_key, value=bound_model, ttl=_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS, @@ -12622,7 +12626,7 @@ class Router: if self._select_pre_routing_strategy(registered_model_name, request_kwargs) is None: await self._delete_claude_code_session_router_binding(cache_key) return registered_model_name - await self.cache.async_set_cache( + await self._claude_code_session_router_cache.async_set_cache( key=cache_key, value=model, ttl=_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS, @@ -13450,6 +13454,9 @@ class Router: def flush_cache(self): litellm.cache = None self.cache.flush_cache() + session_in_memory_cache: Final = self._claude_code_session_router_cache.in_memory_cache + if session_in_memory_cache is not None: + session_in_memory_cache.flush_cache() def reset(self): ## clean up on close diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 77b73a2d12a..a528266d738 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8428,6 +8428,20 @@ class TestClaudeCodeSubagentSessionRouterBinding: assert response is None redis_cache.async_delete_cache.assert_awaited_once() + @pytest.mark.asyncio + async def test_session_bindings_do_not_evict_router_rate_limit_state(self): + router = self._router() + assert router._update_usage(deployment_id="deployment-id", parent_otel_span=None) == 1 + + for session_index in range(201): + request_kwargs = self._request_kwargs() + request_kwargs["proxy_server_request"]["headers"]["X-Claude-Code-Session-Id"] = ( + f"session-{session_index:04d}" + ) + await router.async_pre_routing_hook(model="smart-router", request_kwargs=request_kwargs) + + assert router._update_usage(deployment_id="deployment-id", parent_otel_span=None) == 2 + @pytest.mark.asyncio async def test_background_and_fallback_requests_do_not_clear_the_session_router(self): router = self._router() From 6adc14b4b12250ddc927682c8153469976e87888 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 1 Sep 2026 21:13:45 -0700 Subject: [PATCH 057/175] fix(router): preserve Claude subagent fallbacks --- litellm/router.py | 4 ++-- tests/test_litellm/test_router.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index fb8625bc6ba..6f1f1bc700b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12601,6 +12601,8 @@ class Router: cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs) if cache_key is None or not isinstance(request_kwargs, dict): return registered_model_name + if request_kwargs.get("fallback_depth") not in (None, 0): + return registered_model_name agent_id: Final = self._request_header(request_kwargs, "x-claude-code-agent-id") if agent_id is not None: @@ -12621,8 +12623,6 @@ class Router: if self._request_header(request_kwargs, "x-app") != "cli": return registered_model_name - if request_kwargs.get("fallback_depth") not in (None, 0): - return registered_model_name if self._select_pre_routing_strategy(registered_model_name, request_kwargs) is None: await self._delete_claude_code_session_router_binding(cache_key) return registered_model_name diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index a528266d738..b413bb18f04 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8464,6 +8464,19 @@ class TestClaudeCodeSubagentSessionRouterBinding: assert response is not None assert response.model == "cheap-model" + @pytest.mark.asyncio + async def test_subagent_fallback_does_not_reapply_the_session_router(self): + router = self._router() + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=self._request_kwargs()) + + response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(agent_id="agent-1234", fallback_depth=1), + ) + + assert response is None + @pytest.mark.asyncio async def test_session_router_binding_is_scoped_to_the_authenticated_key(self): router = self._router() From c18511be7d76f0a1dfd18aa07feaa80784afdb9a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:28:38 -0700 Subject: [PATCH 058/175] fix(guardrails): track and tear down presidio sibling callbacks initialize_presidio registers up to three callbacks per guardrail but the registry only kept the first, so deleting or re-syncing the guardrail left the post_call siblings serving the old config. The initializer now returns every callback it registered, the registry tracks primary and siblings per guardrail id, delete purges all of them from every callback list, and update pushes the new params into each while siblings keep their stage. --- .../guardrails/guardrail_initializers.py | 39 ++--- .../proxy/guardrails/guardrail_registry.py | 159 +++++++++++------- .../guardrail_hooks/test_presidio.py | 27 ++- .../guardrails/test_guardrail_registry.py | 129 ++++++++++++++ 4 files changed, 267 insertions(+), 87 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 76dea1b7784..16369abbfb0 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -2,6 +2,7 @@ from typing import Any, Final import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import CommonProxyErrors from litellm.types.guardrails import * @@ -85,7 +86,7 @@ def initialize_lakera_v2(litellm_params: LitellmParams, guardrail: Guardrail): return _lakera_v2_callback -def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail): +def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail) -> tuple[CustomGuardrail, ...]: from litellm.proxy.guardrails.guardrail_hooks.presidio import ( _OPTIONAL_PresidioPIIMasking, ) @@ -94,7 +95,7 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail): run_input: Final = filter_scope in ("input", "both") run_output: Final = filter_scope in ("output", "both") - def _make_presidio_callback(**overrides): + def _make_presidio_callback(**overrides) -> CustomGuardrail: params: Final = dict( guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, @@ -120,27 +121,27 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail): litellm.logging_callback_manager.add_litellm_callback(callback) return callback - primary_callback = None - - if run_input: - primary_callback = _make_presidio_callback() - - if litellm_params.output_parse_pii: - _make_presidio_callback( - output_parse_pii=True, - event_hook=GuardrailEventHooks.post_call.value, - ) - - if run_output: - output_callback: Final = _make_presidio_callback( + input_callback: Final = _make_presidio_callback() if run_input else None + unmask_output_callback: Final = ( + _make_presidio_callback( + output_parse_pii=True, + event_hook=GuardrailEventHooks.post_call.value, + ) + if run_input and litellm_params.output_parse_pii + else None + ) + mask_output_callback: Final = ( + _make_presidio_callback( apply_to_output=True, event_hook=GuardrailEventHooks.post_call.value, output_parse_pii=False, ) - if primary_callback is None: - primary_callback = output_callback - - return primary_callback + if run_output + else None + ) + return tuple( + callback for callback in (input_callback, unmask_output_callback, mask_output_callback) if callback is not None + ) def initialize_hide_secrets(litellm_params: LitellmParams, guardrail: Guardrail): diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index dc13c09dd38..bd35782444b 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -3,10 +3,10 @@ import asyncio import importlib import os -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence from datetime import datetime, timezone from itertools import chain, count -from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, cast +from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, TypeAlias, cast from pydantic import ValidationError @@ -90,6 +90,8 @@ guardrail_initializer_registry: Final = { CONFIG_GUARDRAIL_ID_NAMESPACE: Final = uuid.UUID("625f63f4-935a-50e5-98b5-fbe77babc74a") +GuardrailCallbacks: TypeAlias = tuple[CustomGuardrail, ...] + guardrail_class_registry: Final[dict[str, type[CustomGuardrail]]] = { SupportedGuardrailIntegrations.BEDROCK.value: BedrockGuardrail, SupportedGuardrailIntegrations.GRAYSWAN.value: GraySwanGuardrail, @@ -424,6 +426,41 @@ def _apply_configured_bool_overrides(instance: CustomGuardrail, litellm_params: instance.scan_raw_request = bool(litellm_params.scan_raw_request) +def _as_callback_tuple( + initialized: CustomGuardrail | Sequence[CustomGuardrail] | None, +) -> GuardrailCallbacks: + if initialized is None: + return () + if isinstance(initialized, (list, tuple)): + return tuple(initialized) + return (initialized,) + + +def _configure_callback_scoping( + custom_guardrail_callback: CustomGuardrail, guardrail_name: str, litellm_params: LitellmParams +) -> None: + for scoping_param in ( + "skip_system_message_in_guardrail", + "skip_tool_message_in_guardrail", + "scan_only_tool_results", + ): + setattr(custom_guardrail_callback, scoping_param, getattr(litellm_params, scoping_param, None)) + scan_only_tool_results_enabled: Final = effective_scan_only_tool_results_for_guardrail(custom_guardrail_callback) + if scan_only_tool_results_enabled and not custom_guardrail_callback.supports_scan_only_tool_results(): + raise ValueError( + f"Guardrail {guardrail_name}: scan_only_tool_results is enabled, but this " + "guardrail's role filtering never scans tool results, so no request content would ever " + "be scanned. Remove scan_only_tool_results or the guardrail's role-filtering option." + ) + if scan_only_tool_results_enabled and effective_skip_tool_message_for_guardrail(custom_guardrail_callback): + raise ValueError( + f"Guardrail {guardrail_name}: scan_only_tool_results and " + "skip_tool_message_in_guardrail are enabled together, which excludes every message from " + "scanning, so no request content would ever be scanned. Remove one of the two." + ) + _apply_configured_bool_overrides(custom_guardrail_callback, litellm_params) + + class InMemoryGuardrailHandler: """ Class that handles initializing guardrails and adding them to the CallbackManager @@ -440,6 +477,8 @@ class InMemoryGuardrailHandler: Guardrail id to CustomGuardrail object mapping """ + self.guardrail_id_to_sibling_callbacks: dict[str, GuardrailCallbacks] = {} # mutable-ok: per-id registry + self._sources: dict[str, Literal["db", "config"]] = {} """ Guardrail id to provenance marker. "db" entries are reconciled against @@ -474,7 +513,6 @@ class InMemoryGuardrailHandler: self._sources[guardrail_id] = source return self.IN_MEMORY_GUARDRAILS[guardrail_id] - custom_guardrail_callback: CustomGuardrail | None = None litellm_params_data: Final = guardrail["litellm_params"] verbose_proxy_logger.debug("litellm_params= %s", litellm_params_data) @@ -498,54 +536,15 @@ class InMemoryGuardrailHandler: if guardrail_type is None: raise ValueError("guardrail_type is required") - initializer: Final = guardrail_initializer_registry.get(guardrail_type) - - if initializer: - # Try to call with llm_router first, fall back to without if it fails - import inspect - - sig: Final = inspect.signature(initializer) - if "llm_router" in sig.parameters: - custom_guardrail_callback = initializer( - litellm_params, - guardrail, - llm_router, - ) - else: - custom_guardrail_callback = initializer(litellm_params, guardrail) - elif isinstance(guardrail_type, str) and "." in guardrail_type: - custom_guardrail_callback = self.initialize_custom_guardrail( - guardrail=guardrail, - guardrail_type=guardrail_type, - litellm_params=litellm_params, - config_file_path=config_file_path, - ) - else: - raise ValueError(f"Unsupported guardrail: {guardrail_type}") - - if custom_guardrail_callback is not None: - for scoping_param in ( - "skip_system_message_in_guardrail", - "skip_tool_message_in_guardrail", - "scan_only_tool_results", - ): - setattr(custom_guardrail_callback, scoping_param, getattr(litellm_params, scoping_param, None)) - scan_only_tool_results_enabled: Final = effective_scan_only_tool_results_for_guardrail( - custom_guardrail_callback - ) - if scan_only_tool_results_enabled and not custom_guardrail_callback.supports_scan_only_tool_results(): - raise ValueError( - f"Guardrail {guardrail['guardrail_name']}: scan_only_tool_results is enabled, but this " - "guardrail's role filtering never scans tool results, so no request content would ever " - "be scanned. Remove scan_only_tool_results or the guardrail's role-filtering option." - ) - if scan_only_tool_results_enabled and effective_skip_tool_message_for_guardrail(custom_guardrail_callback): - raise ValueError( - f"Guardrail {guardrail['guardrail_name']}: scan_only_tool_results and " - "skip_tool_message_in_guardrail are enabled together, which excludes every message from " - "scanning, so no request content would ever be scanned. Remove one of the two." - ) - _apply_configured_bool_overrides(custom_guardrail_callback, litellm_params) + created_callbacks: Final = self._create_callbacks( + guardrail=guardrail, + guardrail_type=guardrail_type, + litellm_params=litellm_params, + config_file_path=config_file_path, + llm_router=llm_router, + ) + for custom_guardrail_callback in created_callbacks: + _configure_callback_scoping(custom_guardrail_callback, guardrail["guardrail_name"], litellm_params) parsed_guardrail: Final = Guardrail( guardrail_id=guardrail.get("guardrail_id"), @@ -556,11 +555,44 @@ class InMemoryGuardrailHandler: # store references to the guardrail in memory self.IN_MEMORY_GUARDRAILS[guardrail_id] = parsed_guardrail - self.guardrail_id_to_custom_guardrail[guardrail_id] = custom_guardrail_callback + self.guardrail_id_to_custom_guardrail[guardrail_id] = created_callbacks[0] if created_callbacks else None + self.guardrail_id_to_sibling_callbacks[guardrail_id] = created_callbacks[1:] self._sources[guardrail_id] = source return parsed_guardrail + def _create_callbacks( + self, + guardrail: Guardrail, + guardrail_type: str, + litellm_params: LitellmParams, + config_file_path: str | None, + llm_router: Optional["Router"], + ) -> GuardrailCallbacks: + initializer: Final = guardrail_initializer_registry.get(guardrail_type) + if initializer: + import inspect + + sig: Final = inspect.signature(initializer) + if "llm_router" in sig.parameters: + return _as_callback_tuple(initializer(litellm_params, guardrail, llm_router)) + return _as_callback_tuple(initializer(litellm_params, guardrail)) + if isinstance(guardrail_type, str) and "." in guardrail_type: + return _as_callback_tuple( + self.initialize_custom_guardrail( + guardrail=guardrail, + guardrail_type=guardrail_type, + litellm_params=litellm_params, + config_file_path=config_file_path, + ) + ) + raise ValueError(f"Unsupported guardrail: {guardrail_type}") + + def _tracked_callbacks(self, guardrail_id: str) -> GuardrailCallbacks: + primary: Final = self.guardrail_id_to_custom_guardrail.get(guardrail_id) + siblings: Final = self.guardrail_id_to_sibling_callbacks.get(guardrail_id, ()) + return (() if primary is None else (primary,)) + siblings + def initialize_custom_guardrail( self, guardrail: Guardrail, @@ -630,10 +662,15 @@ class InMemoryGuardrailHandler: self.IN_MEMORY_GUARDRAILS[guardrail_id] = guardrail self._sources[guardrail_id] = source - custom_guardrail_callback: Final = self.guardrail_id_to_custom_guardrail.get(guardrail_id) - if custom_guardrail_callback: - updated_litellm_params: Final = cast(LitellmParams, guardrail.get("litellm_params", {})) - custom_guardrail_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params) + tracked_callbacks: Final = self._tracked_callbacks(guardrail_id) + if not tracked_callbacks: + return + updated_litellm_params: Final = cast(LitellmParams, guardrail.get("litellm_params", {})) + tracked_callbacks[0].update_in_memory_litellm_params(litellm_params=updated_litellm_params) + for sibling_callback in tracked_callbacks[1:]: + sibling_stage = sibling_callback.event_hook + sibling_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params) + sibling_callback.event_hook = sibling_stage def delete_in_memory_guardrail(self, guardrail_id: str) -> None: """ @@ -648,11 +685,11 @@ class InMemoryGuardrailHandler: self.IN_MEMORY_GUARDRAILS.pop(guardrail_id, None) self._sources.pop(guardrail_id, None) - custom_guardrail_callback: Final = self.guardrail_id_to_custom_guardrail.pop(guardrail_id, None) - if custom_guardrail_callback is None: - return - - litellm.logging_callback_manager.remove_callback_from_all_lists(custom_guardrail_callback) + tracked_callbacks: Final = self._tracked_callbacks(guardrail_id) + self.guardrail_id_to_custom_guardrail.pop(guardrail_id, None) + self.guardrail_id_to_sibling_callbacks.pop(guardrail_id, None) + for custom_guardrail_callback in tracked_callbacks: + litellm.logging_callback_manager.remove_callback_from_all_lists(custom_guardrail_callback) def list_in_memory_guardrails(self) -> list[Guardrail]: """ diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 4ee6741ee02..fcf940afd0d 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -842,24 +842,37 @@ async def test_presidio_filter_scope_initializer(monkeypatch): params_input = LitellmParams(guardrail="presidio", mode="pre_call", presidio_filter_scope="input") guardrail_dict = {"guardrail_name": "g1"} - cb = initialize_presidio(params_input, guardrail_dict) - assert cb is created[0] + callbacks = initialize_presidio(params_input, guardrail_dict) + assert callbacks == (created[0],) assert created[0].apply_to_output is False # output-only created.clear() params_output = LitellmParams(guardrail="presidio", mode="pre_call", presidio_filter_scope="output") - cb = initialize_presidio(params_output, guardrail_dict) + callbacks = initialize_presidio(params_output, guardrail_dict) assert len(created) == 1 + assert callbacks == (created[0],) assert created[0].apply_to_output is True - # both -> expect two callbacks (input + output) + # both -> expect two callbacks (input + output), both returned, input first created.clear() params_both = LitellmParams(guardrail="presidio", mode="pre_call", presidio_filter_scope="both") - cb = initialize_presidio(params_both, guardrail_dict) + callbacks = initialize_presidio(params_both, guardrail_dict) assert len(created) == 2 - assert any(not c.apply_to_output for c in created) - assert any(c.apply_to_output for c in created) + assert callbacks == tuple(created) + assert callbacks[0].apply_to_output is False + assert callbacks[1].apply_to_output is True + + # both + output_parse_pii -> three callbacks, all returned, input first + created.clear() + params_all = LitellmParams( + guardrail="presidio", mode="pre_call", presidio_filter_scope="both", output_parse_pii=True + ) + callbacks = initialize_presidio(params_all, guardrail_dict) + assert len(created) == 3 + assert callbacks == tuple(created) + assert callbacks[0].apply_to_output is False + assert mgr.added[-3:] == list(created) @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 2c0735970d3..b8a58f5e3da 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -491,6 +491,135 @@ def test_repeated_db_sync_does_not_accumulate_runner_instances(): cb_list[:] = snapshot +PRESIDIO_SIBLINGS_GID = "55555555-5555-5555-5555-555555555555" +PRESIDIO_SIBLINGS_NAME = "presidio-siblings" + + +def _presidio_db_guardrail(pii_entities_config: dict) -> Guardrail: + return Guardrail( + guardrail_id=PRESIDIO_SIBLINGS_GID, + guardrail_name=PRESIDIO_SIBLINGS_NAME, + litellm_params={ + "guardrail": "presidio", + "mode": "pre_call", + "default_on": True, + "output_parse_pii": True, + "presidio_filter_scope": "both", + "presidio_analyzer_api_base": "https://fakelink.com/v1/presidio/analyze", + "presidio_anonymizer_api_base": "https://fakelink.com/v1/presidio/anonymize", + "pii_entities_config": pii_entities_config, + }, + ) + + +def _presidio_callbacks_in(cb_list) -> list: + return [ + callback + for callback in cb_list + if isinstance(callback, CustomGuardrail) and getattr(callback, "guardrail_name", None) == PRESIDIO_SIBLINGS_NAME + ] + + +def test_presidio_siblings_are_tracked_and_deleted_together(): + """ + A presidio guardrail scoped to both stages registers the pre_call primary plus + the post_call unmask and mask-output siblings. Deleting the guardrail must remove + all three from every callback list, not just the primary. + """ + import litellm + + handler = InMemoryGuardrailHandler() + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + handler.initialize_guardrail(_presidio_db_guardrail({"EMAIL_ADDRESS": "MASK"})) + + registered = _presidio_callbacks_in(litellm.callbacks) + assert len(registered) == 3 + primary = handler.guardrail_id_to_custom_guardrail[PRESIDIO_SIBLINGS_GID] + siblings = handler.guardrail_id_to_sibling_callbacks[PRESIDIO_SIBLINGS_GID] + assert primary is registered[0] + assert siblings == tuple(registered[1:]) + assert [sibling.event_hook for sibling in siblings] == [GuardrailEventHooks.post_call] * 2 + + for cb_list in lists[1:]: + cb_list.extend(registered) + + handler.delete_in_memory_guardrail(PRESIDIO_SIBLINGS_GID) + + for cb_list in lists: + assert _presidio_callbacks_in(cb_list) == [] + assert PRESIDIO_SIBLINGS_GID not in handler.guardrail_id_to_custom_guardrail + assert PRESIDIO_SIBLINGS_GID not in handler.guardrail_id_to_sibling_callbacks + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + +def test_update_in_memory_guardrail_reaches_presidio_siblings_and_keeps_their_stage(): + import litellm + + handler = InMemoryGuardrailHandler() + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + handler.initialize_guardrail(_presidio_db_guardrail({"EMAIL_ADDRESS": "MASK", "IP_ADDRESS": "MASK"})) + tracked = _presidio_callbacks_in(litellm.callbacks) + roles_before = [(callback.apply_to_output, callback.event_hook) for callback in tracked] + + updated = Guardrail( + guardrail_id=PRESIDIO_SIBLINGS_GID, + guardrail_name=PRESIDIO_SIBLINGS_NAME, + litellm_params=LitellmParams( + guardrail="presidio", + mode="pre_call", + default_on=True, + output_parse_pii=True, + presidio_filter_scope="both", + presidio_analyzer_api_base="https://fakelink.com/v1/presidio/analyze", + presidio_anonymizer_api_base="https://fakelink.com/v1/presidio/anonymize", + pii_entities_config={"EMAIL_ADDRESS": "MASK"}, + ), + ) + handler.update_in_memory_guardrail(guardrail_id=PRESIDIO_SIBLINGS_GID, guardrail=updated) + + assert [callback.pii_entities_config for callback in tracked] == [{"EMAIL_ADDRESS": "MASK"}] * 3 + assert [(callback.apply_to_output, callback.event_hook) for callback in tracked] == roles_before + assert _presidio_callbacks_in(litellm.callbacks) == tracked + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + +def test_repeated_db_sync_replaces_presidio_siblings_instead_of_leaking_stale_ones(): + """ + The callback manager dedupes custom loggers by their scalar attributes, so a + leaked post_call sibling blocks the re-initialized sibling from registering and + keeps serving the previous entity config. After every DB re-sync, each callback + list must hold exactly the three current instances, all on the latest config. + """ + import litellm + + handler = InMemoryGuardrailHandler() + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + entity_configs = [{"EMAIL_ADDRESS": "MASK"}, {"EMAIL_ADDRESS": "MASK", "IP_ADDRESS": "MASK"}] + for cycle in range(4): + latest = entity_configs[cycle % 2] + handler.sync_guardrail_from_db(_presidio_db_guardrail(latest)) + for cb_list in lists[1:]: + cb_list.extend(_presidio_callbacks_in(litellm.callbacks)) + + for cb_list in lists: + current = _presidio_callbacks_in(cb_list) + assert len({id(callback) for callback in current}) == 3 + assert all(callback.pii_entities_config == latest for callback in current) + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + def _judge_guardrail(guardrail_id: str) -> Guardrail: return Guardrail( guardrail_id=guardrail_id, From 7cde2cd77f9c39306dddd8c614769e49a507b84b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:39:17 -0700 Subject: [PATCH 059/175] test(guardrails): type the presidio sibling test helpers precisely --- .../test_litellm/proxy/guardrails/test_guardrail_registry.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index b8a58f5e3da..5cbdef5f92f 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -1,3 +1,4 @@ +from collections.abc import Iterable from unittest.mock import AsyncMock, MagicMock import pytest @@ -495,7 +496,7 @@ PRESIDIO_SIBLINGS_GID = "55555555-5555-5555-5555-555555555555" PRESIDIO_SIBLINGS_NAME = "presidio-siblings" -def _presidio_db_guardrail(pii_entities_config: dict) -> Guardrail: +def _presidio_db_guardrail(pii_entities_config: dict[str, str]) -> Guardrail: return Guardrail( guardrail_id=PRESIDIO_SIBLINGS_GID, guardrail_name=PRESIDIO_SIBLINGS_NAME, @@ -512,7 +513,7 @@ def _presidio_db_guardrail(pii_entities_config: dict) -> Guardrail: ) -def _presidio_callbacks_in(cb_list) -> list: +def _presidio_callbacks_in(cb_list: Iterable[object]) -> list[CustomGuardrail]: return [ callback for callback in cb_list From 974b331a4d09e2883d6fe84bb87ce57cc49ab5bb Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 06:15:52 +0000 Subject: [PATCH 060/175] fix: accept persistable router settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/proxy/proxy_server.py | 32 +++++++++----- .../proxy/proxy_server/test_routes_config.py | 43 +++++++++++++++++++ 3 files changed, 65 insertions(+), 11 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 5b44e8b5f51..b6cb6b32187 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -31,6 +31,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( "optional_pre_call_checks", } ) +ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset({"model_list", "search_tools"}) DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dbeb8486539..ae5e08ab566 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -254,6 +254,7 @@ from litellm.constants import ( PROXY_BUDGET_RESCHEDULER_MAX_TIME, PROXY_BUDGET_RESCHEDULER_MIN_TIME, PROXY_CONFIG_RELOAD_INTERVAL_SECONDS, + ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG, USER_SPEND_ALERTS_JOB_ID, WEEKLY_SPEND_REPORT_JOB_ID, ) @@ -5711,13 +5712,9 @@ class ProxyConfig: router_settings: Final = config.get("router_settings", None) if router_settings and isinstance(router_settings, dict): - # model list and search_tools already set - exclude_args: Final = { - "model_list", - "search_tools", - } - - available_args: Final = [x for x in litellm.Router.get_valid_args() if x not in exclude_args] + available_args: Final = [ + x for x in litellm.Router.get_valid_args() if x not in ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG + ] for k, v in router_settings.items(): if k in available_args: @@ -16229,14 +16226,17 @@ async def update_config( ) raw_router_settings: Final = request_body.get("router_settings") if isinstance(raw_router_settings, dict): - unsupported_router_settings: Final = sorted(set(raw_router_settings) - RUNTIME_UPDATABLE_ROUTER_SETTINGS) + supported_router_settings: Final = RUNTIME_UPDATABLE_ROUTER_SETTINGS | ( + frozenset(litellm.Router.get_valid_args()) - ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG + ) + unsupported_router_settings: Final = sorted(set(raw_router_settings) - supported_router_settings) if unsupported_router_settings: raise HTTPException( status_code=400, detail={ "error": ( f"Unsupported router settings: {', '.join(unsupported_router_settings)} " - "are not runtime-updatable router settings" + "are not valid router settings" ) }, ) @@ -16342,10 +16342,20 @@ async def update_config( ) # router_settings: merge existing + request, request wins. - if config_info.router_settings is not None: + if isinstance(raw_router_settings, dict): existing = await _read_section("router_settings") before_router_settings: Final = copy.deepcopy(existing) - updates = config_info.router_settings.dict(exclude_none=True) + typed_router_settings: Final = ( + config_info.router_settings.dict(exclude_none=True) + if config_info.router_settings is not None + else {} + ) + raw_router_settings_without_none: Final = { + key: value + for key, value in raw_router_settings.items() + if key not in typed_router_settings and value is not None + } + updates: Final = {**typed_router_settings, **raw_router_settings_without_none} new_router_settings: Final = {**existing, **updates} await _upsert_section("router_settings", new_router_settings) asyncio.create_task( diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index 4b0954c350a..6166513d229 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -81,6 +81,49 @@ def test_config_update_persists_optional_pre_call_checks(client, auth_as, mock_p assert persisted["optional_pre_call_checks"] == ["prompt_caching"] +def test_config_update_persists_model_group_affinity_config(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + fake_proxy_config = MagicMock() + fake_proxy_config.add_deployment = AsyncMock() + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + model_group_affinity_config = {"gpt-4": ["session_affinity"]} + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"model_group_affinity_config": model_group_affinity_config}}, + ) + + assert response.status_code == 200 + persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"]) + assert persisted["model_group_affinity_config"] == model_group_affinity_config + + +def test_config_update_persists_disable_cooldowns(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + fake_proxy_config = MagicMock() + fake_proxy_config.add_deployment = AsyncMock() + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"disable_cooldowns": True}}, + ) + + assert response.status_code == 200 + persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"]) + assert persisted["disable_cooldowns"] is True + + def test_config_update_rejects_unknown_router_setting(client, auth_as, mock_prisma, monkeypatch): from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles From 95e09db661471d8fbdaa1a93f79bf8e49ecdd0eb Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 06:56:27 +0000 Subject: [PATCH 061/175] style: apply ruff format to router settings merge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ae5e08ab566..1bdf2ba9987 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16346,9 +16346,7 @@ async def update_config( existing = await _read_section("router_settings") before_router_settings: Final = copy.deepcopy(existing) typed_router_settings: Final = ( - config_info.router_settings.dict(exclude_none=True) - if config_info.router_settings is not None - else {} + config_info.router_settings.dict(exclude_none=True) if config_info.router_settings is not None else {} ) raw_router_settings_without_none: Final = { key: value From aba9644297e4e709232775031cb03f028dd1cbd6 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 07:10:04 +0000 Subject: [PATCH 062/175] fix: avoid router settings update name collision Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1bdf2ba9987..6632b209905 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16353,8 +16353,8 @@ async def update_config( for key, value in raw_router_settings.items() if key not in typed_router_settings and value is not None } - updates: Final = {**typed_router_settings, **raw_router_settings_without_none} - new_router_settings: Final = {**existing, **updates} + router_settings_updates: Final = {**typed_router_settings, **raw_router_settings_without_none} + new_router_settings: Final = {**existing, **router_settings_updates} await _upsert_section("router_settings", new_router_settings) asyncio.create_task( create_config_audit_log( From 385957e830c6b5edae9455dae8361527908cd4bb Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 07:49:25 +0000 Subject: [PATCH 063/175] fix: reject constructor-managed router settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 11 +++++- .../proxy/proxy_server/test_routes_config.py | 36 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/litellm/constants.py b/litellm/constants.py index b6cb6b32187..1c1939bd350 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -31,7 +31,16 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( "optional_pre_call_checks", } ) -ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset({"model_list", "search_tools"}) +ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset( + { + "model_list", + "search_tools", + "assistants_config", + "router_general_settings", + "ignore_invalid_deployments", + "fallback_access_check", + } +) DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index 6166513d229..dcb63b8ca82 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -124,6 +124,42 @@ def test_config_update_persists_disable_cooldowns(client, auth_as, mock_prisma, assert persisted["disable_cooldowns"] is True +def test_config_update_rejects_assistants_config(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"assistants_config": {"enabled": True}}}, + ) + + assert response.status_code == 400 + assert "assistants_config" in response.json()["error"]["message"] + table.upsert.assert_not_called() + + +def test_config_update_rejects_router_general_settings(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"router_general_settings": {"async_only_mode": True}}}, + ) + + assert response.status_code == 400 + assert "router_general_settings" in response.json()["error"]["message"] + table.upsert.assert_not_called() + + def test_config_update_rejects_unknown_router_setting(client, auth_as, mock_prisma, monkeypatch): from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles From 1400070d711f645290fb382564e1f21200c5e610 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 07:58:09 +0000 Subject: [PATCH 064/175] chore(techdebt): clear fresh debt from the 2026-09-01 window Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- basedpyright-code-budget.json | 6 +++--- .../integrations/SlackAlerting/slack_alerting.py | 9 ++++----- .../websearch_interception/handler.py | 1 - .../_experimental/mcp_server/rest_endpoints.py | 13 +++++++++---- .../guardrails/guardrail_hooks/alice/alice.py | 16 ++++++++-------- type-discipline-budget.json | 6 +++--- 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 2d4adc02234..f14f8e002dd 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5601 }, "reportMissingTypeArgument": { - "limit": 15300 + "limit": 15298 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,13 +105,13 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38347 + "limit": 38344 }, "reportUnknownParameterType": { "limit": 19626 }, "reportUnknownVariableType": { - "limit": 29884 + "limit": 29880 }, "reportUnnecessaryCast": { "limit": 111 diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 748ef938cea..dc41c7dadc8 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1955,11 +1955,10 @@ Model Info: if not thresholds_enabled and not anomalies_enabled: return - if prisma_client is None: - from litellm.proxy.proxy_server import prisma_client as global_prisma_client + from litellm.proxy.proxy_server import prisma_client as global_prisma_client - prisma_client = global_prisma_client # rebind-ok: fall back to the proxy's global client - if prisma_client is None: + client: Final = prisma_client if prisma_client is not None else global_prisma_client + if client is None: return from litellm.integrations.SlackAlerting.user_spend_alerts import ( @@ -1970,7 +1969,7 @@ Model Info: try: today: Final = datetime.datetime.now(datetime.timezone.utc).date() rows: Final = await fetch_user_spend_rows( - prisma_client=prisma_client, + prisma_client=client, today=today, baseline_days=self.alerting_args.spend_anomaly_baseline_days, ) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index dc61ee38a8c..2d737bc34e7 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -419,7 +419,6 @@ class WebSearchInterceptionLogger(CustomLogger): if call_type in (CallTypes.responses, CallTypes.aresponses): return self._convert_responses_tools(kwargs=kwargs, tools=tools) - # Check if any tool is a web search tool (native or already LiteLLM standard) has_websearch: Final = any(is_web_search_tool(t) for t in tools) if not has_websearch: diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index d1ef73a15cd..90474bfc5e6 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -92,6 +92,9 @@ def _connection_error_message(exc: BaseException) -> str: if MCP_AVAILABLE: + from mcp.types import Tool as MCPTool + + from litellm.experimental_mcp_client.client import MCPClient from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, global_mcp_server_manager, @@ -876,7 +879,6 @@ if MCP_AVAILABLE: return (), classify_list_exception(e) return tools_result, ServerListOk(tool_count=len(tools_result)) - # Query all servers the user has access to queried_servers: Final = tuple( server for server in map(global_mcp_server_manager.get_mcp_server_by_id, allowed_server_ids) @@ -1141,6 +1143,11 @@ if MCP_AVAILABLE: scopes: Final[list[str] | None] = scopes_raw if isinstance(scopes_raw, list) else None return client_id, client_secret, scopes + async def _list_tools_within(client: MCPClient, deadline: float) -> list[MCPTool] | None: + with anyio.move_on_after(deadline): + return await client.list_tools(raise_on_error=True) + return None + async def _execute_with_mcp_client( request: NewMCPServerRequest, operation: Callable[..., Awaitable[Mapping[str, object]]], @@ -1422,9 +1429,7 @@ if MCP_AVAILABLE: getattr(client, "timeout", MCP_CLIENT_TIMEOUT) or MCP_CLIENT_TIMEOUT, MCP_TOOL_LISTING_TIMEOUT, ) - list_tools_result = None # rebind-ok: set inside the timeout scope below - with anyio.move_on_after(listing_deadline): - list_tools_result = await client.list_tools(raise_on_error=True) # rebind-ok: fills the init above + list_tools_result: Final = await _list_tools_within(client, listing_deadline) if list_tools_result is None: verbose_logger.warning( "MCP tools/list preview timed out after %s seconds while paginating upstream tools", diff --git a/litellm/proxy/guardrails/guardrail_hooks/alice/alice.py b/litellm/proxy/guardrails/guardrail_hooks/alice/alice.py index 27018769909..9cabac2d0fa 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/alice/alice.py +++ b/litellm/proxy/guardrails/guardrail_hooks/alice/alice.py @@ -8,6 +8,7 @@ import json import os from collections.abc import Mapping +from itertools import islice from typing import ( TYPE_CHECKING, Any, # noqa: TID251 # **kwargs forwards verbatim to CustomGuardrail.__init__; see ruff-strict.toml @@ -341,19 +342,18 @@ def _json_safe( if depth >= _MAX_DEPTH or id(value) in seen: return None - nested: Final = seen | {id(value)} # mutable-ok: one-shot set literal, unioned into a frozenset immediately + nested: Final = seen | frozenset((id(value),)) if isinstance(value, dict): - out: dict[str, object] = {} # mutable-ok: bounded accumulator local to this call, never escapes as-is - for key, item in list(value.items())[:_MAX_ITEMS]: # mutable-ok: list() only to slice an unordered view - if isinstance(key, str) and key not in strip_keys: - out[key] = _json_safe(item, depth + 1, nested, strip_keys) - return out + return { + key: _json_safe(item, depth + 1, nested, strip_keys) + for key, item in islice(value.items(), _MAX_ITEMS) + if isinstance(key, str) and key not in strip_keys + } if isinstance(value, (list, tuple, set, frozenset)): return [ # mutable-ok: return value is a one-shot list, discarded by the caller after use - _json_safe(item, depth + 1, nested, strip_keys) - for item in list(value)[:_MAX_ITEMS] # mutable-ok: list() only to slice an unordered view + _json_safe(item, depth + 1, nested, strip_keys) for item in islice(value, _MAX_ITEMS) ] dump: Final = getattr(value, "model_dump", None) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index fbf998533f8..0f39b32670a 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -6,7 +6,7 @@ "limit": 26777 }, "LIT003": { - "limit": 266 + "limit": 265 }, "LIT004": { "limit": 40 @@ -27,10 +27,10 @@ "limit": 0 }, "LIT010": { - "limit": 16504 + "limit": 16502 }, "LIT011": { - "limit": 5531 + "limit": 5529 }, "LIT012": { "limit": 4495 From a7836ede15bb4f62d8f44bdb991402a9829727e3 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 14:48:51 +0000 Subject: [PATCH 065/175] fix(models): absorb open registry PRs: govcloud bedrock and mantle, azure gov, openai tiered long-context, scaleway, together qwen3.8, azure ai cache and kimi k2.7 code, azure mai deprecations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 605 +++++++++++++++++- model_prices_and_context_window.json | 605 +++++++++++++++++- .../llm_cost_calc/test_llm_cost_calc_utils.py | 6 +- ...penai_service_tier_long_context_pricing.py | 156 +++++ whitelisted_bedrock_models.txt | 14 + 5 files changed, 1341 insertions(+), 45 deletions(-) create mode 100644 tests/test_litellm/test_openai_service_tier_long_context_pricing.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c710db1a749..87d348f4752 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9643,7 +9643,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2.5-Flash": { "input_cost_per_image_token": 1.75e-06, @@ -9656,7 +9657,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2e": { "deprecation_date": "2026-08-15", @@ -10155,7 +10157,9 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "cache_read_input_token_cost": 1.45e-07, + "supports_prompt_caching": true }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", @@ -10169,18 +10173,20 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "cache_read_input_token_cost": 2.8e-08, + "supports_prompt_caching": true }, "azure_ai/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, + "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.1e-07, + "output_cost_per_token": 1.32e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_prompt_caching": true, @@ -10400,11 +10406,13 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supports_function_calling": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true }, "azure_ai/kimi-k2.6": { "deprecation_date": "2027-04-16", @@ -10415,7 +10423,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k2-6-in-microsoft-foundry/4513125", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supported_modalities": [ "text", "image" @@ -10426,7 +10434,9 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1.6e-07, + "supports_prompt_caching": true }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, @@ -12110,7 +12120,7 @@ "max_output_tokens": 2048, "max_tokens": 2048, "mode": "chat", - "output_cost_per_token": 2.65e-06, + "output_cost_per_token": 6e-07, "supports_pdf_input": true }, "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": { @@ -29098,16 +29108,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29119,6 +29132,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29161,16 +29175,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29182,6 +29199,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29225,16 +29243,19 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, "cache_creation_input_token_cost_flex": 1.25e-06, "cache_creation_input_token_cost_priority": 5e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, @@ -29246,6 +29267,7 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, "output_cost_per_token_above_272k_tokens_flex": 9e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, @@ -29288,16 +29310,19 @@ "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, "cache_creation_input_token_cost_flex": 1.25e-07, "cache_creation_input_token_cost_priority": 5e-07, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_above_272k_tokens_flex": 2e-07, + "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, @@ -29309,6 +29334,7 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, "output_cost_per_token_above_272k_tokens_flex": 9e-07, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, @@ -29548,7 +29574,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, @@ -29602,7 +29631,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -29751,7 +29783,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, @@ -29800,7 +29835,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, @@ -29849,7 +29887,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, @@ -29898,7 +29938,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, @@ -41350,13 +41392,13 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/Qwen/Qwen3.8-2.4T-A95B": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, "max_tokens": 1010000, "mode": "chat", - "output_cost_per_token": 6.25e-06, + "output_cost_per_token": 6e-06, "source": "https://docs.together.ai/docs/serverless-models", "supports_prompt_caching": true }, @@ -57558,5 +57600,526 @@ "supported_endpoints": [ "/v1/audio/transcriptions" ] + }, + "scaleway/glm-5.2": { + "input_cost_per_token": 1.8e-06, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 5.5e-06, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": false + }, + "scaleway/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": false + }, + "azure_ai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "deprecation_date": "2026-10-03", + "input_cost_per_token": 9.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-west-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_above_272k_tokens": 5.28e-06, + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-06, + "cache_read_input_token_cost": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-07, + "output_cost_per_token": 1.584e-05, + "output_cost_per_token_above_272k_tokens": 2.376e-05 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-07, + "input_cost_per_token_above_272k_tokens": 5.28e-07, + "cache_creation_input_token_cost": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-07, + "cache_read_input_token_cost": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-08, + "output_cost_per_token": 1.584e-06, + "output_cost_per_token_above_272k_tokens": 2.376e-06 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.3": { + "use_openai_responses_path": true, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/", + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.4e-07 + }, + "bedrock_mantle/us-gov-east-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "azure/us-gov/gpt-5.1": { + "cache_read_input_token_cost": 1.71875e-07, + "default_reasoning_effort": "none", + "deprecation_date": "2027-05-15", + "input_cost_per_token": 1.71875e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.375e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us-gov/o3-mini": { + "cache_read_input_token_cost": 7.57e-07, + "deprecation_date": "2026-10-01", + "input_cost_per_token": 1.513e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 6.05e-06, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "azure/us-gov/text-embedding-3-large": { + "deprecation_date": "2028-02-09", + "input_cost_per_token": 1.63e-07, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "azure/us-gov/text-embedding-3-small": { + "deprecation_date": "2028-02-09", + "input_cost_per_token": 2.5e-08, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c710db1a749..87d348f4752 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9643,7 +9643,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2.5-Flash": { "input_cost_per_image_token": 1.75e-06, @@ -9656,7 +9657,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2e": { "deprecation_date": "2026-08-15", @@ -10155,7 +10157,9 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "cache_read_input_token_cost": 1.45e-07, + "supports_prompt_caching": true }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", @@ -10169,18 +10173,20 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "cache_read_input_token_cost": 2.8e-08, + "supports_prompt_caching": true }, "azure_ai/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, + "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.1e-07, + "output_cost_per_token": 1.32e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_prompt_caching": true, @@ -10400,11 +10406,13 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supports_function_calling": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true }, "azure_ai/kimi-k2.6": { "deprecation_date": "2027-04-16", @@ -10415,7 +10423,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k2-6-in-microsoft-foundry/4513125", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supported_modalities": [ "text", "image" @@ -10426,7 +10434,9 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1.6e-07, + "supports_prompt_caching": true }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, @@ -12110,7 +12120,7 @@ "max_output_tokens": 2048, "max_tokens": 2048, "mode": "chat", - "output_cost_per_token": 2.65e-06, + "output_cost_per_token": 6e-07, "supports_pdf_input": true }, "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": { @@ -29098,16 +29108,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29119,6 +29132,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29161,16 +29175,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29182,6 +29199,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29225,16 +29243,19 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, "cache_creation_input_token_cost_flex": 1.25e-06, "cache_creation_input_token_cost_priority": 5e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, @@ -29246,6 +29267,7 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, "output_cost_per_token_above_272k_tokens_flex": 9e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, @@ -29288,16 +29310,19 @@ "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, "cache_creation_input_token_cost_flex": 1.25e-07, "cache_creation_input_token_cost_priority": 5e-07, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_above_272k_tokens_flex": 2e-07, + "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, @@ -29309,6 +29334,7 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, "output_cost_per_token_above_272k_tokens_flex": 9e-07, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, @@ -29548,7 +29574,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, @@ -29602,7 +29631,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -29751,7 +29783,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, @@ -29800,7 +29835,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, @@ -29849,7 +29887,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, @@ -29898,7 +29938,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, @@ -41350,13 +41392,13 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/Qwen/Qwen3.8-2.4T-A95B": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, "max_tokens": 1010000, "mode": "chat", - "output_cost_per_token": 6.25e-06, + "output_cost_per_token": 6e-06, "source": "https://docs.together.ai/docs/serverless-models", "supports_prompt_caching": true }, @@ -57558,5 +57600,526 @@ "supported_endpoints": [ "/v1/audio/transcriptions" ] + }, + "scaleway/glm-5.2": { + "input_cost_per_token": 1.8e-06, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 5.5e-06, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": false + }, + "scaleway/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": false + }, + "azure_ai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "deprecation_date": "2026-10-03", + "input_cost_per_token": 9.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-west-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_above_272k_tokens": 5.28e-06, + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-06, + "cache_read_input_token_cost": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-07, + "output_cost_per_token": 1.584e-05, + "output_cost_per_token_above_272k_tokens": 2.376e-05 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-07, + "input_cost_per_token_above_272k_tokens": 5.28e-07, + "cache_creation_input_token_cost": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-07, + "cache_read_input_token_cost": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-08, + "output_cost_per_token": 1.584e-06, + "output_cost_per_token_above_272k_tokens": 2.376e-06 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.3": { + "use_openai_responses_path": true, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/", + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.4e-07 + }, + "bedrock_mantle/us-gov-east-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "azure/us-gov/gpt-5.1": { + "cache_read_input_token_cost": 1.71875e-07, + "default_reasoning_effort": "none", + "deprecation_date": "2027-05-15", + "input_cost_per_token": 1.71875e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.375e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us-gov/o3-mini": { + "cache_read_input_token_cost": 7.57e-07, + "deprecation_date": "2026-10-01", + "input_cost_per_token": 1.513e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 6.05e-06, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "azure/us-gov/text-embedding-3-large": { + "deprecation_date": "2028-02-09", + "input_cost_per_token": 1.63e-07, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "azure/us-gov/text-embedding-3-small": { + "deprecation_date": "2028-02-09", + "input_cost_per_token": 2.5e-08, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 } } diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 0e1c832ebf5..5c8de19a7e9 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1522,7 +1522,7 @@ def test_gpt_5_6_alias_prices_match_sol(local_model_cost_map): sol = litellm.model_cost["gpt-5.6-sol"] cost_fields = sorted(field for field in sol if "cost" in field) - assert len(cost_fields) == 23 + assert len(cost_fields) == 27 for field in cost_fields: assert alias.get(field) == sol.get(field), field @@ -4039,8 +4039,8 @@ def test_fast_service_tier_matches_priority_above_the_context_threshold(_local_m ) assert fast == priority - assert fast[0] == pytest.approx(300_000 * 8e-06, rel=1e-9) - assert fast[1] == pytest.approx(1_000 * 3e-05, rel=1e-9) + assert fast[0] == pytest.approx(300_000 * 1.6e-05, rel=1e-9) + assert fast[1] == pytest.approx(1_000 * 6e-05, rel=1e-9) def test_priority_reasoning_tokens_bill_at_the_priority_output_rate(_local_model_cost_map): diff --git a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py new file mode 100644 index 00000000000..c0860a5b55f --- /dev/null +++ b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py @@ -0,0 +1,156 @@ +import json +from functools import lru_cache +from pathlib import Path + +import pytest + +import litellm + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +FLEX_LONG_CONTEXT = { + "gpt-5.4": { + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07, + }, + "gpt-5.4-pro": { + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135, + }, + "gpt-5.5": { + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07, + }, +} + +PRIORITY_LONG_CONTEXT = { + "gpt-5.6": { + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, + }, + "gpt-5.6-sol": { + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, + }, + "gpt-5.6-terra": { + "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + }, + "gpt-5.6-luna": { + "input_cost_per_token_above_272k_tokens_priority": 8e-07, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, + }, +} + +EXPECTED = {**FLEX_LONG_CONTEXT, **PRIORITY_LONG_CONTEXT} + +NO_PUBLISHED_PRIORITY_LONG_CONTEXT = ("gpt-5.4", "gpt-5.5") + + +@pytest.fixture(autouse=True) +def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + +@lru_cache(maxsize=2) +def _load(path: Path) -> dict[str, dict[str, object]]: + with open(path) as f: + return json.load(f) + + +@pytest.mark.parametrize("path", [MAIN_PATH, BACKUP_PATH], ids=["main", "backup"]) +@pytest.mark.parametrize("model", sorted(EXPECTED)) +def test_service_tier_long_context_rates_are_published(model: str, path: Path) -> None: + """Each tier must carry its own above-272K rates, in both price files.""" + info = _load(path).get(model) + assert info is not None, f"{model} not found in {path.name}" + for key, expected in EXPECTED[model].items(): + assert info.get(key) == pytest.approx(expected), f"{model}.{key} is {info.get(key)!r}, expected {expected!r}" + + +@pytest.mark.parametrize("model", sorted(EXPECTED)) +def test_tier_long_context_rate_is_half_or_double_the_standard(model: str) -> None: + """Flex is half the standard long-context rate; priority is double it.""" + info = _load(MAIN_PATH)[model] + tier = "flex" if model in FLEX_LONG_CONTEXT else "priority" + ratio = 0.5 if tier == "flex" else 2.0 + for base in ("input_cost_per_token", "output_cost_per_token"): + standard = info[f"{base}_above_272k_tokens"] + tiered = info[f"{base}_above_272k_tokens_{tier}"] + assert tiered == pytest.approx(standard * ratio), ( + f"{model}.{base}_above_272k_tokens_{tier} is {tiered!r}, " + f"expected {ratio}x the standard long-context rate {standard!r}" + ) + + +@pytest.mark.parametrize("model", NO_PUBLISHED_PRIORITY_LONG_CONTEXT) +def test_no_priority_long_context_rates_where_openai_publishes_none(model: str) -> None: + """Guard against back-filling a rate OpenAI does not publish.""" + info = _load(MAIN_PATH)[model] + assert "input_cost_per_token_above_272k_tokens_priority" not in info + + +LONG_CONTEXT_PROMPT_TOKENS = 300_000 +COMPLETION_TOKENS = 1_000 + +TIERED_COST_CASES = [ + ("gpt-5.4", "flex", 2.5e-06, 1.125e-05), + ("gpt-5.4-pro", "flex", 3e-05, 0.000135), + ("gpt-5.5", "flex", 5e-06, 2.25e-05), + ("gpt-5.6", "priority", 1.6e-05, 6e-05), + ("gpt-5.6-sol", "priority", 1.6e-05, 6e-05), + ("gpt-5.6-terra", "priority", 8e-06, 3.6e-05), + ("gpt-5.6-luna", "priority", 8e-07, 3.6e-06), +] + + +@pytest.mark.parametrize("model,tier,input_rate,output_rate", TIERED_COST_CASES) +def test_cost_per_token_bills_long_context_at_the_tier_rate( + model: str, tier: str, input_rate: float, output_rate: float +) -> None: + """A prompt over 272K on flex or priority must bill at that tier's long-context rate.""" + input_cost, output_cost = litellm.cost_per_token( + model=model, + prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + service_tier=tier, + ) + assert input_cost == pytest.approx(LONG_CONTEXT_PROMPT_TOKENS * input_rate) + assert output_cost == pytest.approx(COMPLETION_TOKENS * output_rate) + + +@pytest.mark.parametrize("model,tier,input_rate,output_rate", TIERED_COST_CASES) +def test_cost_per_token_tier_differs_from_the_standard_long_context_cost( + model: str, tier: str, input_rate: float, output_rate: float +) -> None: + """Flex halves the standard long-context bill and priority doubles it.""" + ratio = 0.5 if tier == "flex" else 2.0 + standard = sum( + litellm.cost_per_token( + model=model, + prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + ) + ) + tiered = sum( + litellm.cost_per_token( + model=model, + prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + service_tier=tier, + ) + ) + assert tiered == pytest.approx(standard * ratio) diff --git a/whitelisted_bedrock_models.txt b/whitelisted_bedrock_models.txt index 7e20081988d..8753d7c3c77 100644 --- a/whitelisted_bedrock_models.txt +++ b/whitelisted_bedrock_models.txt @@ -217,3 +217,17 @@ bedrock/us-east-1/zai.glm-5 bedrock/us-west-2/zai.glm-5 bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0 bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0 +bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b +bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2 +bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b +bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0 +bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0 +bedrock/us-gov-west-1/anthropic.claude-sonnet-5 +bedrock/us-gov-west-1/anthropic.claude-opus-4-8 +bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b +bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2 +bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b +bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0 +bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0 +bedrock/us-gov-east-1/anthropic.claude-sonnet-5 +bedrock/us-gov-east-1/anthropic.claude-opus-4-8 From 8588a2ea42f7fac2a19e37123ebac5a7327b182a Mon Sep 17 00:00:00 2001 From: Oliver Jensen Date: Wed, 2 Sep 2026 17:01:54 +0200 Subject: [PATCH 066/175] fix(docker): install saml extra in litellm-backend image (#39291) The monolithic images install the saml extra but the split backend image did not, so /sso/saml/* returned 501 on Helm split-image deployments. The gateway image is unchanged since /sso/ routes are backend-only. --- backend/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/Dockerfile b/backend/Dockerfile index aa01b9fba8b..622fedcd70d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -46,6 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ + --extra saml \ --python python3.13 # Stage 2 — copy source and install the project + workspace members. @@ -57,6 +58,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ + --extra saml \ --python python3.13 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ From 6b83b16559e5ceb4904121bcb90623a5f9f7115c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 08:04:14 -0700 Subject: [PATCH 067/175] feat(gemini): day-0 pricing for gemini-3.8-flash Gemini 3.8 Flash launches today with the same promotional pricing, limits, and thinking settings as Gemini 3.7 Flash, so the gemini/, vertex_ai/, and bare cost map entries mirror the 3.7 Flash ones. Regression tests lock the launch prices, the 4096-token cache minimum, and the gemini-3 thought signature gate in for the new model. --- ...odel_prices_and_context_window_backup.json | 173 ++++++++++++++++++ model_prices_and_context_window.json | 173 ++++++++++++++++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 45 +++++ .../test_vertex_ai_gemini_transformation.py | 3 + tests/test_litellm/test_utils.py | 1 + 5 files changed, 395 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a3cfb300ea6..cc828e126ad 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23514,6 +23514,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "vertex_ai/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -25351,6 +25408,65 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "rpm": 2000, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "tpm": 800000, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -25759,6 +25875,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_audio_token": 7e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a3cfb300ea6..cc828e126ad 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23514,6 +23514,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "vertex_ai/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -25351,6 +25408,65 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "rpm": 2000, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "tpm": 800000, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -25759,6 +25875,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_audio_token": 7e-07, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 0e1c832ebf5..0ccb05c67a3 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -4200,6 +4200,51 @@ def test_generic_cost_per_token_gemini_37_flash(_local_model_cost_map): assert completion_cost == pytest.approx(0.001875) +GEMINI_38_FLASH_LAUNCH_PRICING = [ + ("gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08), + ("gemini/gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08), + ("vertex_ai/gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08), +] + + +@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_38_FLASH_LAUNCH_PRICING) +def test_gemini_38_flash_launch_pricing(model, input_cost, output_cost, cache_read_cost, _local_model_cost_map): + model_cost_map = litellm.model_cost[model] + assert model_cost_map["input_cost_per_token"] == input_cost + assert model_cost_map["output_cost_per_token"] == output_cost + assert model_cost_map["output_cost_per_reasoning_token"] == output_cost + assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost + assert model_cost_map["mode"] == "chat" + assert model_cost_map["supports_reasoning"] is True + assert model_cost_map["supports_function_calling"] is True + assert model_cost_map["max_input_tokens"] == 1048576 + + +def test_gemini_38_flash_matches_37_flash_promotional_pricing(_local_model_cost_map): + for prefix in ("", "gemini/", "vertex_ai/"): + assert litellm.model_cost[f"{prefix}gemini-3.8-flash"] == litellm.model_cost[f"{prefix}gemini-3.7-flash"] + + +def test_generic_cost_per_token_gemini_38_flash(_local_model_cost_map): + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=200, + text_tokens=300, + ), + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=1000), + ) + prompt_cost, completion_cost = generic_cost_per_token( + model="gemini-3.8-flash", + usage=usage, + custom_llm_provider="gemini", + ) + assert prompt_cost == pytest.approx(0.00075) + assert completion_cost == pytest.approx(0.001875) + + def test_grok_46_launch_pricing(_local_model_cost_map): model_cost_map = litellm.model_cost["xai/grok-4.6"] assert model_cost_map["input_cost_per_token"] == 2e-06 diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 8c1de12e7d9..4679b978f78 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -1096,10 +1096,13 @@ def test_natively_signed_parallel_turn_never_carries_a_placeholder(model): "gemini-3.5-flash", "gemini-3.6-flash", "gemini-3.7-flash", + "gemini-3.8-flash", "vertex_ai/gemini-3.5-flash", "vertex_ai/gemini-3.7-flash", + "vertex_ai/gemini-3.8-flash", "gemini/gemini-3.5-flash", "gemini/gemini-3.7-flash", + "gemini/gemini-3.8-flash", ], ) def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model): diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 521e91daded..0790b41c349 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4655,6 +4655,7 @@ GEMINI_4096_CACHE_MIN_MODELS: Final = tuple( "gemini-3.5-flash", "gemini-3.6-flash", "gemini-3.7-flash", + "gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3.1-pro-preview-customtools", ) From b76127774059d577229bbc9f74b3bf1b9fef812c Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 15:09:49 +0000 Subject: [PATCH 068/175] fix(models): drop inherited retirement dates from azure/us-gov entries pending a Government schedule source Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 ---- model_prices_and_context_window.json | 4 ---- 2 files changed, 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 87d348f4752..63b88c2a7b4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -58055,7 +58055,6 @@ "azure/us-gov/gpt-5.1": { "cache_read_input_token_cost": 1.71875e-07, "default_reasoning_effort": "none", - "deprecation_date": "2027-05-15", "input_cost_per_token": 1.71875e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -58090,7 +58089,6 @@ }, "azure/us-gov/o3-mini": { "cache_read_input_token_cost": 7.57e-07, - "deprecation_date": "2026-10-01", "input_cost_per_token": 1.513e-06, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -58105,7 +58103,6 @@ "supports_vision": false }, "azure/us-gov/text-embedding-3-large": { - "deprecation_date": "2028-02-09", "input_cost_per_token": 1.63e-07, "litellm_provider": "azure", "max_input_tokens": 8191, @@ -58114,7 +58111,6 @@ "output_cost_per_token": 0.0 }, "azure/us-gov/text-embedding-3-small": { - "deprecation_date": "2028-02-09", "input_cost_per_token": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 8191, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 87d348f4752..63b88c2a7b4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -58055,7 +58055,6 @@ "azure/us-gov/gpt-5.1": { "cache_read_input_token_cost": 1.71875e-07, "default_reasoning_effort": "none", - "deprecation_date": "2027-05-15", "input_cost_per_token": 1.71875e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -58090,7 +58089,6 @@ }, "azure/us-gov/o3-mini": { "cache_read_input_token_cost": 7.57e-07, - "deprecation_date": "2026-10-01", "input_cost_per_token": 1.513e-06, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -58105,7 +58103,6 @@ "supports_vision": false }, "azure/us-gov/text-embedding-3-large": { - "deprecation_date": "2028-02-09", "input_cost_per_token": 1.63e-07, "litellm_provider": "azure", "max_input_tokens": 8191, @@ -58114,7 +58111,6 @@ "output_cost_per_token": 0.0 }, "azure/us-gov/text-embedding-3-small": { - "deprecation_date": "2028-02-09", "input_cost_per_token": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 8191, From 07cf9dc46f5a4fd3b506f89cc77744f723eff190 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 08:10:52 -0700 Subject: [PATCH 069/175] feat(models): add Azure DeepSeek V4 Flash 0731 --- .../model_prices_and_context_window_backup.json | 16 ++++++++++++++++ model_prices_and_context_window.json | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a3cfb300ea6..893ac49f5c7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -10171,6 +10171,22 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-12-03", + "input_cost_per_token": 1.9e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.1e-07, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "azure_ai/deepseek-v4-flash-0731": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-12-03", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a3cfb300ea6..893ac49f5c7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -10171,6 +10171,22 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-12-03", + "input_cost_per_token": 1.9e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.1e-07, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "azure_ai/deepseek-v4-flash-0731": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-12-03", From da23e0241dc82649ff56f2e73e6e156c4e098129 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 15:28:53 +0000 Subject: [PATCH 070/175] fix(models): add cloudflare whisper transcription pricing and pin govcloud pricing tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 20 ++ model_prices_and_context_window.json | 20 ++ .../test_bedrock_usgov_pricing.py | 200 +++++++++++++++--- ...st_cloudflare_workers_ai_model_metadata.py | 16 ++ 4 files changed, 230 insertions(+), 26 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 63b88c2a7b4..30621a17df3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -58117,5 +58117,25 @@ "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/openai/whisper": { + "input_cost_per_second": 7.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "cloudflare/@cf/openai/whisper-large-v3-turbo": { + "input_cost_per_second": 8.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper-large-v3-turbo/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 63b88c2a7b4..30621a17df3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -58117,5 +58117,25 @@ "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/openai/whisper": { + "input_cost_per_second": 7.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "cloudflare/@cf/openai/whisper-large-v3-turbo": { + "input_cost_per_second": 8.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper-large-v3-turbo/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] } } diff --git a/tests/test_litellm/test_bedrock_usgov_pricing.py b/tests/test_litellm/test_bedrock_usgov_pricing.py index 6b3312b5cc4..f9e8fd4c46c 100644 --- a/tests/test_litellm/test_bedrock_usgov_pricing.py +++ b/tests/test_litellm/test_bedrock_usgov_pricing.py @@ -26,9 +26,7 @@ import pytest @pytest.fixture(scope="module") def model_data(): - json_path = os.path.join( - os.path.dirname(__file__), "../../model_prices_and_context_window.json" - ) + json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json") with open(json_path) as f: return json.load(f) @@ -51,21 +49,14 @@ def test_usgov_sonnet_4_5_pricing(model_data, model_key): info = model_data[model_key] assert info["input_cost_per_token"] == 3.6e-06, ( - f"{model_key}: input_cost_per_token should be $3.60/MTok " - f"(got {info['input_cost_per_token']})" + f"{model_key}: input_cost_per_token should be $3.60/MTok (got {info['input_cost_per_token']})" ) - assert ( - info["output_cost_per_token"] == 1.8e-05 - ), f"{model_key}: output_cost_per_token should be $18.00/MTok" - assert ( - info["cache_creation_input_token_cost"] == 4.5e-06 - ), f"{model_key}: 5m cache write should be $4.50/MTok" - assert ( - info["cache_creation_input_token_cost_above_1hr"] == 7.2e-06 - ), f"{model_key}: 1h cache write should be $7.20/MTok" - assert ( - info["cache_read_input_token_cost"] == 3.6e-07 - ), f"{model_key}: cache read should be $0.36/MTok" + assert info["output_cost_per_token"] == 1.8e-05, f"{model_key}: output_cost_per_token should be $18.00/MTok" + assert info["cache_creation_input_token_cost"] == 4.5e-06, f"{model_key}: 5m cache write should be $4.50/MTok" + assert info["cache_creation_input_token_cost_above_1hr"] == 7.2e-06, ( + f"{model_key}: 1h cache write should be $7.20/MTok" + ) + assert info["cache_read_input_token_cost"] == 3.6e-07, f"{model_key}: cache read should be $0.36/MTok" def test_usgov_carries_20_percent_premium_over_global(model_data): @@ -84,9 +75,7 @@ def test_usgov_carries_20_percent_premium_over_global(model_data): "cache_read_input_token_cost", ): ratio = usgov_info[field] / global_info[field] - assert ( - abs(ratio - 1.2) < 1e-9 - ), f"{field}: us-gov / global ratio is {ratio}, expected 1.2" + assert abs(ratio - 1.2) < 1e-9, f"{field}: us-gov / global ratio is {ratio}, expected 1.2" # The us-gov.anthropic.* cross-region inference profile is the only us-gov @@ -112,9 +101,7 @@ def test_usgov_cross_region_above_200k_carries_gov_premium(model_data, field, ex """ info = model_data[USGOV_CROSS_REGION_KEY] assert field in info, f"{USGOV_CROSS_REGION_KEY}: missing field {field}" - assert ( - info[field] == expected - ), f"{USGOV_CROSS_REGION_KEY}: {field} should be {expected} (got {info[field]})" + assert info[field] == expected, f"{USGOV_CROSS_REGION_KEY}: {field} should be {expected} (got {info[field]})" def test_usgov_cross_region_above_200k_ratio_to_global(model_data): @@ -127,6 +114,167 @@ def test_usgov_cross_region_above_200k_ratio_to_global(model_data): usgov_info = model_data[USGOV_CROSS_REGION_KEY] for field in EXPECTED_USGOV_ABOVE_200K: ratio = usgov_info[field] / global_info[field] - assert ( - abs(ratio - 1.2) < 1e-9 - ), f"{field}: us-gov / global ratio is {ratio}, expected 1.2" + assert abs(ratio - 1.2) < 1e-9, f"{field}: us-gov / global ratio is {ratio}, expected 1.2" + + +CLAUDE_GOV_EXPECTED = { + "anthropic.claude-sonnet-5": { + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 1.2e-05, + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + }, + "anthropic.claude-opus-4-8": { + "input_cost_per_token": 6e-06, + "output_cost_per_token": 3e-05, + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + }, +} + + +@pytest.mark.parametrize("base_key", CLAUDE_GOV_EXPECTED) +@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"]) +def test_usgov_claude_sonnet5_opus48_pricing(model_data, region, base_key): + """Sonnet 5 and Opus 4.8 gov entries must match the rates AWS publishes + for both GovCloud regions on the Bedrock pricing page (1.2x global). + """ + gov_key = f"bedrock/{region}/{base_key}" + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + for field, expected in CLAUDE_GOV_EXPECTED[base_key].items(): + assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})" + ratio = info[field] / model_data[base_key][field] + assert abs(ratio - 1.2) < 1e-9, f"{gov_key}: {field} gov/global ratio is {ratio}, expected 1.2" + + +CONVERSE_GOV_EXPECTED = { + "nvidia.nemotron-nano-3-30b": (7.2e-08, 2.88e-07), + "nvidia.nemotron-nano-12b-v2": (2.4e-07, 7.2e-07), + "nvidia.nemotron-super-3-120b": (1.8e-07, 7.8e-07), + "openai.gpt-oss-20b-1:0": (8.4e-08, 3.6e-07), + "openai.gpt-oss-120b-1:0": (1.8e-07, 7.2e-07), +} + + +@pytest.mark.parametrize("base_key", CONVERSE_GOV_EXPECTED) +@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"]) +def test_usgov_converse_model_pricing(model_data, region, base_key): + """Nemotron and gpt-oss gov entries must match the AWS Bedrock offer file, + which prices both GovCloud regions identically at 1.2x commercial. + """ + gov_key = f"bedrock/{region}/{base_key}" + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + expected_input, expected_output = CONVERSE_GOV_EXPECTED[base_key] + assert info["input_cost_per_token"] == expected_input + assert info["output_cost_per_token"] == expected_output + assert info["litellm_provider"] == "bedrock" + base = model_data[base_key] + assert abs(info["input_cost_per_token"] / base["input_cost_per_token"] - 1.2) < 1e-9 + assert abs(info["output_cost_per_token"] / base["output_cost_per_token"] - 1.2) < 1e-9 + + +def test_usgov_west_llama3_8b_output_price_fixed(model_data): + """The us-gov-west-1 llama3-8b entry carried the 70B output rate ($2.65/MTok); + the AWS Bedrock offer file prices output at $0.60/MTok. AWS lists the model + in us-gov-west-1 only, so there is no east entry to check. + """ + info = model_data["bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0"] + assert info["input_cost_per_token"] == 3e-07 + assert info["output_cost_per_token"] == 6e-07 + + +MANTLE_GOV_TIERED_EXPECTED = { + "openai.gpt-5.6-luna": { + "input_cost_per_token": 2.64e-07, + "input_cost_per_token_above_272k_tokens": 5.28e-07, + "cache_creation_input_token_cost": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-07, + "cache_read_input_token_cost": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-08, + "output_cost_per_token": 1.584e-06, + "output_cost_per_token_above_272k_tokens": 2.376e-06, + }, + "openai.gpt-5.6-terra": { + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_above_272k_tokens": 5.28e-06, + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-06, + "cache_read_input_token_cost": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-07, + "output_cost_per_token": 1.584e-05, + "output_cost_per_token_above_272k_tokens": 2.376e-05, + }, +} + + +@pytest.mark.parametrize("model", MANTLE_GOV_TIERED_EXPECTED) +def test_usgov_west_mantle_terra_luna_pricing(model_data, model): + """Terra and Luna carry 1.2x commercial across every tier in the + us-gov-west-1 offer file; the us-gov-east-1 offer file has no SKUs for them. + """ + gov_key = f"bedrock_mantle/us-gov-west-1/{model}" + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + for field, expected in MANTLE_GOV_TIERED_EXPECTED[model].items(): + assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})" + assert info["litellm_provider"] == "bedrock_mantle" + assert f"bedrock_mantle/us-gov-east-1/{model}" not in model_data + + +@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"]) +def test_usgov_mantle_gpt_5_4_pricing_has_no_long_context_tier(model_data, region): + """gpt-5.4 gov rates come from the offer file, which publishes only the + standard tier in GovCloud: no long-context SKUs exist there, unlike commercial. + """ + gov_key = f"bedrock_mantle/{region}/openai.gpt-5.4" + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + assert info["input_cost_per_token"] == 3.3e-06 + assert info["cache_read_input_token_cost"] == 3.3e-07 + assert info["output_cost_per_token"] == 1.98e-05 + assert not any(field.endswith("_above_272k_tokens") for field in info) + + +def test_usgov_mantle_grok_4_3_west_only(model_data): + """grok-4.3 is priced in the us-gov-west-1 offer file only; the east offer + file carries grok-4.6 instead. + """ + info = model_data["bedrock_mantle/us-gov-west-1/xai.grok-4.3"] + assert info["input_cost_per_token"] == 1.5e-06 + assert info["output_cost_per_token"] == 3e-06 + assert info["cache_read_input_token_cost"] == 2.4e-07 + assert "bedrock_mantle/us-gov-east-1/xai.grok-4.3" not in model_data + + +AZURE_GOV_EXPECTED = { + "azure/us-gov/gpt-5.1": { + "input_cost_per_token": 1.71875e-06, + "cache_read_input_token_cost": 1.71875e-07, + "output_cost_per_token": 1.375e-05, + }, + "azure/us-gov/o3-mini": { + "input_cost_per_token": 1.513e-06, + "cache_read_input_token_cost": 7.57e-07, + "output_cost_per_token": 6.05e-06, + }, + "azure/us-gov/text-embedding-3-large": {"input_cost_per_token": 1.63e-07}, + "azure/us-gov/text-embedding-3-small": {"input_cost_per_token": 2.5e-08}, +} + + +@pytest.mark.parametrize("gov_key", AZURE_GOV_EXPECTED) +def test_azure_usgov_pricing(model_data, gov_key): + """Azure Government meters from the Azure retail prices API + (usgovvirginia/usgovarizona, serviceName 'Foundry Models'). No Government + retirement schedule is published, so these entries carry no deprecation_date. + """ + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + for field, expected in AZURE_GOV_EXPECTED[gov_key].items(): + assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})" + assert info["litellm_provider"] == "azure" + assert "deprecation_date" not in info diff --git a/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py b/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py index 9ca4515239a..e33bcfb8378 100644 --- a/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py +++ b/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py @@ -75,6 +75,22 @@ def test_additional_current_models_are_present(): assert entry["output_cost_per_token"] > 0 +@pytest.mark.parametrize( + "key, published_price_per_audio_minute", + [ + ("cloudflare/@cf/openai/whisper", 0.00045), + ("cloudflare/@cf/openai/whisper-large-v3-turbo", 0.00051), + ], +) +def test_whisper_transcription_pricing_is_stored_per_second(key, published_price_per_audio_minute): + entry = litellm.model_cost[key] + assert entry["litellm_provider"] == "cloudflare" + assert entry["mode"] == "audio_transcription" + assert entry["supported_endpoints"] == ["/v1/audio/transcriptions"] + assert entry["output_cost_per_second"] == 0.0 + assert entry["input_cost_per_second"] == pytest.approx(published_price_per_audio_minute / 60) + + def test_root_and_backup_have_identical_cloudflare_keys(): if not os.path.exists(ROOT_MAP): pytest.skip("root cost map only ships in source checkouts") From 2ce4e3f8a99e12efce9433640059d9fca7bfb448 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 08:32:49 -0700 Subject: [PATCH 071/175] fix(guardrails): run apply_guardrail-only providers in logging_only mode (#39297) * fix(guardrails): run apply_guardrail-only providers in logging_only mode A CustomGuardrail that implements only apply_guardrail inherited the CustomLogger no-op async_logging_hook, so mode: logging_only never scanned anything and never recorded guardrail_information. CustomGuardrail.async_logging_hook now routes the logged request and response through the call type's guardrail translation on copies and appends the verdict to standard_logging_object.guardrail_information. Resolves LIT-4876 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): keep logging_only scan copies inside the error boundary and return a fresh logging payload Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): cover embedding scan, native-hook bypass, and unmapped call type in logging_only Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 65 ++++++ .../integrations/test_custom_guardrail.py | 199 ++++++++++++++++++ 2 files changed, 264 insertions(+) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index e87ac9521ae..372c9bf6b91 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1,4 +1,5 @@ import contextvars +import copy import hashlib import os import secrets @@ -39,6 +40,7 @@ except ImportError: if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation dc: Final = DualCache() @@ -852,6 +854,69 @@ class CustomGuardrail(CustomLogger): return result + async def async_logging_hook( + self, + kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract + result: object, + call_type: str, + ) -> tuple[dict, object]: # mutable-ok: CustomLogger.async_logging_hook contract + """logging_only: run apply_guardrail on copies of the logged request/response and record the verdict.""" + from litellm.llms import get_guardrail_translation_mapping + + if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks: + return kwargs, result + try: + translation: Final = get_guardrail_translation_mapping(CallTypes(call_type))() + except ValueError: + verbose_logger.debug( + "Guardrail %s: no guardrail translation for call_type=%s, skipping logging_only scan", + self.guardrail_name, + call_type, + ) + return kwargs, result + litellm_params: Final = kwargs.get("litellm_params") or {} + scratch_metadata: Final = { + key: value + for key, value in (litellm_params.get("metadata") or {}).items() + if key != "standard_logging_guardrail_information" + } + try: + await self._scan_logged_call(kwargs, result, translation, scratch_metadata) + except Exception as e: + verbose_logger.warning("Guardrail %s: logging_only scan raised: %s", self.guardrail_name, e) + recorded: Final = scratch_metadata.get("standard_logging_guardrail_information") + standard_logging_object: Final = kwargs.get("standard_logging_object") + if not recorded or not isinstance(standard_logging_object, dict): + return kwargs, result + entries: Final = recorded if isinstance(recorded, list) else [recorded] + existing: Final = standard_logging_object.get("guardrail_information") or [] + return { + **kwargs, + "standard_logging_object": {**standard_logging_object, "guardrail_information": [*existing, *entries]}, + }, result + + async def _scan_logged_call( + self, + kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract + result: object, + translation: "BaseTranslation", + scratch_metadata: dict, # mutable-ok: apply_guardrail records its verdict into request metadata + ) -> None: + optional_params: Final = kwargs.get("optional_params") or {} + scratch_input: Final = copy.deepcopy(kwargs.get("messages") or kwargs.get("input")) + scratch_request: Final = { + "model": kwargs.get("model"), + "messages": scratch_input, + "input": scratch_input, + "tools": copy.deepcopy(optional_params.get("tools")), + "litellm_call_id": kwargs.get("litellm_call_id"), + "metadata": scratch_metadata, + } + await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self) + await translation.process_output_response( + response=copy.deepcopy(result), guardrail_to_apply=self, request_data=scratch_request + ) + def supports_scan_only_tool_results(self) -> bool: """Whether this guardrail can scan tool-result content. diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index d978eb48c12..7d70b9a8862 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -2237,3 +2237,202 @@ class TestRecordsOwnGuardrailInformation: ) assert _guardrail_entries(request_data) == [] + + +class _ApplyOnlyObserver(CustomGuardrail): + """Overrides only apply_guardrail, like panw_prisma_airs; inherits async_logging_hook.""" + + def __init__(self, block: bool = False): + from litellm.types.guardrails import GuardrailEventHooks + + super().__init__(guardrail_name="apply-only-observer", event_hook=GuardrailEventHooks.logging_only) + self.block = block + self.calls: list = [] + + @log_guardrail_information + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + from fastapi import HTTPException + + self.calls.append((input_type, list(inputs.get("texts") or []))) + if self.block: + raise HTTPException(status_code=400, detail={"error": "flagged"}) + return GenericGuardrailAPIInputs(texts=["[MASKED]" for _ in inputs.get("texts") or []]) + + +def _logged_call(messages: list | str) -> tuple[dict, object]: + from litellm.types.utils import Choices, Message, ModelResponse + + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="general kenobi"))]) + kwargs = { + "model": "gpt-5.4-mini", + "messages": messages, + "litellm_call_id": "call-1", + "litellm_params": {"metadata": {"user_api_key_user_id": "u1"}}, + "optional_params": {}, + "standard_logging_object": {"guardrail_information": None}, + } + return kwargs, response + + +class TestLoggingOnlyApplyGuardrail: + """LIT-4876 regression: a guardrail in mode logging_only that implements only + apply_guardrail must still run against the logged request and response and + record guardrail_information, instead of inheriting the CustomLogger no-op.""" + + @pytest.mark.asyncio + async def test_runs_apply_guardrail_observe_only_and_records_verdict(self): + guardrail = _ApplyOnlyObserver() + messages = [{"role": "user", "content": "hello there"}] + kwargs, response = _logged_call(messages) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])] + assert out_kwargs["messages"] == [{"role": "user", "content": "hello there"}] + assert out_response.choices[0].message.content == "general kenobi" + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_name"] for e in entries] == ["apply-only-observer", "apply-only-observer"] + assert {e["guardrail_mode"] for e in entries} == {"logging_only"} + assert {e["guardrail_status"] for e in entries} == {"success"} + assert "standard_logging_guardrail_information" not in kwargs["litellm_params"]["metadata"] + assert kwargs["standard_logging_object"] == {"guardrail_information": None} + + @pytest.mark.asyncio + async def test_appends_to_pre_call_verdicts_without_duplicating_them(self): + guardrail = _ApplyOnlyObserver() + kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) + pre_call_entry = {"guardrail_name": "pii-blocker", "guardrail_mode": "pre_call", "guardrail_status": "success"} + kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] = [pre_call_entry] + kwargs["standard_logging_object"]["guardrail_information"] = [pre_call_entry] + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_name"] for e in entries] == ["pii-blocker", "apply-only-observer", "apply-only-observer"] + assert kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] == [pre_call_entry] + + @pytest.mark.asyncio + async def test_request_copy_failure_is_swallowed(self): + import threading + + guardrail = _ApplyOnlyObserver() + kwargs, response = _logged_call([{"role": "user", "content": "hello there", "lock": threading.Lock()}]) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [] + assert out_kwargs is kwargs + assert out_response is response + + @pytest.mark.asyncio + async def test_block_verdict_is_recorded_without_raising(self): + guardrail = _ApplyOnlyObserver(block=True) + kwargs, response = _logged_call([{"role": "user", "content": "flagged content"}]) + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [("request", ["flagged content"])] + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["guardrail_intervened"] + + @pytest.mark.asyncio + async def test_call_type_without_translation_is_skipped(self): + guardrail = _ApplyOnlyObserver() + kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.amoderation.value) + + assert guardrail.calls == [] + assert out_kwargs["standard_logging_object"]["guardrail_information"] is None + + @pytest.mark.asyncio + async def test_aembedding_scans_logged_input(self): + from litellm.types.utils import EmbeddingResponse + + guardrail = _ApplyOnlyObserver() + kwargs, _ = _logged_call("hello there") + response = EmbeddingResponse(data=[{"embedding": [0.1], "index": 0, "object": "embedding"}]) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.aembedding.value) + + assert guardrail.calls == [("request", ["hello there"])] + assert out_kwargs["messages"] == "hello there" + assert out_response is response + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["success"] + + @pytest.mark.asyncio + async def test_native_lifecycle_hook_guardrail_is_left_alone(self): + class _NativeHooks(_ApplyOnlyObserver): + use_native_lifecycle_hooks = True + + guardrail = _NativeHooks() + kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [] + assert out_kwargs is kwargs + assert out_response is response + + @pytest.mark.asyncio + async def test_aresponses_scans_logged_messages_when_input_is_cleared(self): + from litellm.types.llms.openai import ResponsesAPIResponse + + guardrail = _ApplyOnlyObserver() + kwargs, _ = _logged_call([{"role": "user", "content": "hello there"}]) + kwargs["input"] = None + response = ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="gpt-5.4-mini", + object="response", + status="completed", + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "general kenobi"}], + } + ], + ) + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.aresponses.value) + + assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])] + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["success", "success"] + + @pytest.mark.asyncio + async def test_async_success_handler_records_verdict_in_standard_logging_object(self): + import datetime as dt + + from litellm.litellm_core_utils.litellm_logging import Logging + + guardrail = _ApplyOnlyObserver() + guardrail.default_on = True + messages = [{"role": "user", "content": "hello there"}] + _, response = _logged_call(messages) + logging_obj = Logging( + model="gpt-5.4-mini", + messages=messages, + stream=False, + call_type=CallTypes.acompletion.value, + start_time=dt.datetime.now(), + litellm_call_id="call-1", + function_id="fn-1", + dynamic_async_success_callbacks=[guardrail], + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {}}, optional_params={}, model="gpt-5.4-mini", custom_llm_provider="openai" + ) + + await logging_obj.async_success_handler( + result=response, start_time=dt.datetime.now(), end_time=dt.datetime.now() + ) + + assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])] + entries = logging_obj.model_call_details["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["success", "success"] From 69cd1bada6249889a9412155a6086faea65bfe6c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 08:46:28 -0700 Subject: [PATCH 072/175] test(gemini): compare gemini-3.8-flash to 3.7 flash field by field --- .../llm_cost_calc/test_llm_cost_calc_utils.py | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 0ccb05c67a3..9b3e60764e3 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -4220,9 +4220,44 @@ def test_gemini_38_flash_launch_pricing(model, input_cost, output_cost, cache_re assert model_cost_map["max_input_tokens"] == 1048576 -def test_gemini_38_flash_matches_37_flash_promotional_pricing(_local_model_cost_map): - for prefix in ("", "gemini/", "vertex_ai/"): - assert litellm.model_cost[f"{prefix}gemini-3.8-flash"] == litellm.model_cost[f"{prefix}gemini-3.7-flash"] +GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH = ( + "input_cost_per_token", + "output_cost_per_token", + "output_cost_per_reasoning_token", + "cache_read_input_token_cost", + "input_cost_per_token_batches", + "output_cost_per_token_batches", + "input_cost_per_token_flex", + "output_cost_per_token_flex", + "cache_read_input_token_cost_flex", + "input_cost_per_token_priority", + "output_cost_per_token_priority", + "cache_read_input_token_cost_priority", + "search_context_cost_per_query", + "google_maps_grounding_cost_per_query", + "prompt_cache_min_tokens", + "max_input_tokens", + "max_output_tokens", + "supports_reasoning", + "supports_function_calling", + "supports_prompt_caching", + "supports_vision", + "supports_pdf_input", + "supports_audio_input", + "supports_video_input", + "supports_response_schema", + "supports_tool_choice", + "supports_web_search", + "supports_url_context", +) + + +@pytest.mark.parametrize("prefix", ["", "gemini/", "vertex_ai/"]) +def test_gemini_38_flash_matches_37_flash_promotional_pricing(prefix, _local_model_cost_map): + new_model = litellm.model_cost[f"{prefix}gemini-3.8-flash"] + old_model = litellm.model_cost[f"{prefix}gemini-3.7-flash"] + for field in GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH: + assert new_model[field] == old_model[field], field def test_generic_cost_per_token_gemini_38_flash(_local_model_cost_map): From 7b919f89a85ea7671fd0c3ac2fb27d31fb74201b Mon Sep 17 00:00:00 2001 From: moe-berri Date: Wed, 2 Sep 2026 10:00:47 -0700 Subject: [PATCH 073/175] fix(router): track routed model in fallback attempts --- .../router_utils/fallback_event_handlers.py | 5 +-- .../test_fallback_event_handlers.py | 21 +++++++++++ tests/test_litellm/test_router.py | 35 +++++++++++++++++-- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 3d37ca216a7..0167721f9fe 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -470,10 +470,11 @@ async def run_async_fallback( attempted: Final = ( carried_targets if isinstance(carried_targets, AttemptedFallbackTargets) else AttemptedFallbackTargets() ) - attempted.record(original_model_group) + failed_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group + attempted.record(failed_model_group) for mg in fallback_model_group: - if mg == original_model_group: + if mg == failed_model_group: continue if same_model_group_only and _get_fallback_target_model_group(mg) != original_model_group: verbose_router_logger.info( diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 894b2d9e74f..9e51a60364b 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -614,6 +614,27 @@ async def test_run_async_fallback_forwards_attempted_model_groups_to_nested_call ) +@pytest.mark.asyncio +async def test_run_async_fallback_can_target_the_requested_group_when_a_pre_router_replaced_it(): + """The requested group was never called when a pre-router selected a tier, so a + tier fallback may legitimately target that originally requested group.""" + router = RecordingRouter() + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["requested-model"], + original_model_group="requested-model", + original_exception=RuntimeError("selected tier failed"), + max_fallbacks=3, + fallback_depth=0, + model="requested-model", + metadata={"pre_routing_selected_model": "selected-tier"}, + ) + + assert router.received_kwargs["model"] == "requested-model" + assert router.received_kwargs["attempted_targets"].keys == frozenset({"selected-tier", "requested-model"}) + + @pytest.mark.asyncio @pytest.mark.parametrize( "entry", diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index b413bb18f04..d255135bf64 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8335,20 +8335,26 @@ class TestClaudeCodeSubagentSessionRouterBinding: ) @classmethod - def _router(cls) -> "litellm.Router": + def _router( + cls, + cheap_response: str = "cheap response", + fallbacks: list[dict[str, list[str]]] | None = None, + ) -> "litellm.Router": from litellm.types.router import TaggedPreRoutingStrategy router = litellm.Router( model_list=[ { "model_name": "cheap-model", - "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "cheap response"}, + "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": cheap_response}, }, { "model_name": "expensive-model", "litellm_params": {"model": "openai/gpt-4o", "mock_response": "expensive response"}, }, - ] + ], + fallbacks=fallbacks, + num_retries=0, ) router.complexity_routers = { "smart-router": [TaggedPreRoutingStrategy(tags=(), strategy=cls._RewriteStrategy())] @@ -8477,6 +8483,29 @@ class TestClaudeCodeSubagentSessionRouterBinding: assert response is None + @pytest.mark.asyncio + async def test_subagent_can_fallback_to_its_original_requested_model(self): + router = self._router( + cheap_response="litellm.RateLimitError", + fallbacks=[{"cheap-model": ["expensive-model"]}], + ) + + await router.acompletion( + model="smart-router", + messages=[{"role": "user", "content": "main turn"}], + **self._request_kwargs(), + ) + subagent_kwargs = self._request_kwargs(agent_id="agent-1234") + + response = await router.acompletion( + model="expensive-model", + messages=[{"role": "user", "content": "subagent turn"}], + **subagent_kwargs, + ) + + assert response.choices[0].message.content == "expensive response" + assert subagent_kwargs["metadata"]["routing_decision"]["routed_model"] == "cheap-model" + @pytest.mark.asyncio async def test_session_router_binding_is_scoped_to_the_authenticated_key(self): router = self._router() From de80e3afe448237c69a535517ca88c12d079ee6d Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 2 Sep 2026 10:19:03 -0700 Subject: [PATCH 074/175] fix(helm): scale the classic chart's HPA out at the documented 60 percent CPU (#35975) * fix(helm): scale the classic chart's HPA out at the documented 60 percent CPU The litellm-helm chart shipped targetCPUUtilizationPercentage: 80, which is unexamined helm create scaffold rather than a chosen number. It arrived packaged with the stock minReplicas: 1, maxReplicas: 100, a commented-out targetMemoryUtilizationPercentage: 80, and the boilerplate "such as Minikube" comment, the same provenance as the 128Mi resource example this file just corrected. 60 is the documented recommendation. The mechanism behind it is scale-up lag: the chart's own startupProbe is failureThreshold: 30 times periodSeconds: 10, so a replica can take up to 300 seconds to become ready, and a pod added at 80 percent utilization arrives minutes after saturation. The memory target stays commented out on purpose. The prisma query engine's resident memory is a high-water mark that ratchets to the pod's worst-ever write and is never returned, so a memory-target HPA reads the largest write a pod ever did rather than what it is doing now, and replicas ratchet up without scaling back in. hpa_tests.yaml carried its second suite after a YAML document separator, and helm-unittest loads only the first document per file, so that suite never ran; an assertion planted in it still passed. Fold it into the one live suite and add coverage pinning the rendered CPU target, the absence of a memory metric by default, and that overrides still take effect. Bump the chart to 1.1.2, since rendered output changes for anyone running with autoscaling enabled. * fix(helm): bump litellm-helm to 1.1.3 after rebase onto 1.1.2 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- helm/litellm-helm/Chart.yaml | 2 +- helm/litellm-helm/tests/hpa_tests.yaml | 42 ++++++++++++++++++++++---- helm/litellm-helm/values.yaml | 11 ++++++- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/helm/litellm-helm/Chart.yaml b/helm/litellm-helm/Chart.yaml index 3959d85edf3..a3cb388ffc6 100644 --- a/helm/litellm-helm/Chart.yaml +++ b/helm/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.1.2 +version: 1.1.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/litellm-helm/tests/hpa_tests.yaml b/helm/litellm-helm/tests/hpa_tests.yaml index ec18c3591d3..cd062dd5971 100644 --- a/helm/litellm-helm/tests/hpa_tests.yaml +++ b/helm/litellm-helm/tests/hpa_tests.yaml @@ -1,4 +1,4 @@ -suite: "hpa with behavior" +suite: "hpa" templates: - hpa.yaml tests: @@ -23,14 +23,44 @@ tests: - equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 } - equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 } ---- -suite: "hpa without behavior" -templates: - - hpa.yaml -tests: - it: "does not render behavior when not set" set: autoscaling.enabled: true asserts: - isKind: { of: HorizontalPodAutoscaler } - isNull: { path: spec.behavior } + + - it: "scales on cpu at the documented 60 percent by default" + set: + autoscaling.enabled: true + asserts: + - isKind: { of: HorizontalPodAutoscaler } + - equal: { path: "spec.metrics[0].resource.name", value: cpu } + - equal: { path: "spec.metrics[0].resource.target.type", value: Utilization } + - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 } + + - it: "does not scale on memory by default" + set: + autoscaling.enabled: true + asserts: + - lengthEqual: { path: spec.metrics, count: 1 } + + - it: "honours an explicit cpu target override" + set: + autoscaling.enabled: true + autoscaling.targetCPUUtilizationPercentage: 75 + asserts: + - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 } + + - it: "renders a memory metric only when a memory target is set" + set: + autoscaling.enabled: true + autoscaling.targetMemoryUtilizationPercentage: 80 + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - equal: { path: "spec.metrics[1].resource.name", value: memory } + - equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 } + + - it: "renders no hpa when autoscaling is disabled" + asserts: + - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index f8df98de102..637be2322e3 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -200,7 +200,16 @@ autoscaling: enabled: false minReplicas: 1 maxReplicas: 100 - targetCPUUtilizationPercentage: 80 + # 60 is the documented recommendation. See "Recommended Machine Specifications" + # in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe + # above only after up to failureThreshold x periodSeconds = 300 seconds, so a target + # high enough to trip near saturation adds capacity minutes after it was needed. + targetCPUUtilizationPercentage: 60 + # Deliberately left unset rather than given a value. The prisma query engine's + # resident memory is a high-water mark that ratchets to the pod's worst-ever write + # and is never returned, so a memory target reads the largest write a pod ever did + # rather than what it is doing now, and replicas ratchet up without scaling back in. + # Memory is a floor to provision under 'resources', not a signal to scale on. # targetMemoryUtilizationPercentage: 80 # behavior: {} From dba190842cea106ab4b03880861038ddbe6aae42 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:26:59 -0700 Subject: [PATCH 075/175] fix(anthropic): keep the cache_control normalizer inside the type-discipline budget --- litellm/llms/anthropic/common_utils.py | 38 +++++++++++++++++--------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index b5bfb32c0c6..19d3d6d7043 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1404,13 +1404,33 @@ def _with_portable_cache_control_in_message(message: object) -> object: return message return { # mutable-ok: JSON wire format **message, - "content": [_with_portable_cache_control_in_content_block(block) for block in content], + "content": [ # mutable-ok: JSON wire format + _with_portable_cache_control_in_content_block(block) for block in content + ], } -def normalize_cache_control_in_anthropic_payload( # mutable-ok: JSON wire format +def _with_portable_cache_control_in_messages(messages: object) -> object: + if isinstance(messages, str) or not isinstance(messages, Sequence): + return messages + return [ # mutable-ok: JSON wire format + _with_portable_cache_control_in_message(message) for message in messages + ] + + +def _with_portable_cache_control_in_scoped_value(key: str, value: object) -> object: + match key: + case "system" | "tools": + return _with_portable_cache_control_in_blocks(value) + case "messages": + return _with_portable_cache_control_in_messages(value) + case _: + return value + + +def normalize_cache_control_in_anthropic_payload( payload: Mapping[str, object], -) -> dict[str, object]: +) -> dict[str, object]: # mutable-ok: JSON wire format """ Return a copy of an Anthropic /v1/messages payload with every ``cache_control`` entry reduced to ``{"type": }`` @@ -1427,17 +1447,9 @@ def normalize_cache_control_in_anthropic_payload( # mutable-ok: JSON wire forma dropped entirely. The caller's payload is never mutated. """ portable: Final = _with_portable_cache_control(payload) - scoped: Final = { # mutable-ok: JSON wire format - key: ( - _with_portable_cache_control_in_blocks(value) - if key in ("system", "tools") - else [_with_portable_cache_control_in_message(message) for message in value] - if key == "messages" and isinstance(value, Sequence) and not isinstance(value, str) - else value - ) - for key, value in portable.items() + return { # mutable-ok: JSON wire format + key: _with_portable_cache_control_in_scoped_value(key, value) for key, value in portable.items() } - return scoped def process_anthropic_headers(headers: httpx.Headers | dict) -> dict: From 53da9bca8e45af86507cb6b5c83736290913ba71 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:29:44 -0700 Subject: [PATCH 076/175] fix(bedrock): drop client_metadata for every converse model --- .../bedrock/chat/converse_transformation.py | 10 +-- litellm/llms/bedrock/common_utils.py | 9 -- .../chat/test_converse_transformation.py | 85 +++---------------- 3 files changed, 15 insertions(+), 89 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 38b9569856a..df52b78f6b5 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -86,7 +86,6 @@ from litellm.utils import ( from ..common_utils import ( BedrockError, BedrockModelInfo, - bedrock_arn_hides_model_family, bedrock_converse_supports_parallel_tool_use_config, bedrock_model_accepts_cache_points, get_anthropic_beta_from_headers, @@ -1335,14 +1334,7 @@ class AmazonConverseConfig(BaseConfig): ) additional_request_params.pop("parallel_tool_calls", None) - - drops_client_metadata: Final = base_model.startswith("anthropic") or bedrock_arn_hides_model_family(model) - if drops_client_metadata and additional_request_params.pop("client_metadata", None) is not None: - litellm.verbose_logger.debug( - "Bedrock Converse: dropping `client_metadata` for model=%s, Anthropic rejects it with " - "'client_metadata: Extra inputs are not permitted'", - model, - ) + additional_request_params.pop("client_metadata", None) # Only set the topK value in for models that support it additional_request_params.update(self._handle_top_k_value(model, inference_params, drop_params)) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index cf18e3e3ec8..66ee5f10679 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -809,15 +809,6 @@ def get_bedrock_base_model(model: str) -> str: return model -def bedrock_arn_hides_model_family(model: str) -> bool: - """ - True for an ARN-addressed model whose base name carries no ``provider.model`` - id, such as an application inference profile or a provisioned throughput ARN. - Callers that gate behavior on the model family cannot resolve one here. - """ - return "arn:" in model.lower() and "." not in get_bedrock_base_model(model) - - def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool: return any( (litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 7556e1624be..fb165c38ef9 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -979,16 +979,19 @@ def test_config_blocks_do_not_leak_into_inference_config(): assert data["serviceTier"] == {"type": "priority"} -@pytest.mark.parametrize("model", ["anthropic.claude-opus-4-8", "us.anthropic.claude-opus-4-8"]) -def test_client_metadata_stripped_for_anthropic_converse_request(model): - """``client_metadata`` sent by codex must not reach Anthropic as a passthrough model field. - - Converse forwards ``additionalModelRequestFields`` verbatim to the model, and Anthropic - rejects the request with "client_metadata: Extra inputs are not permitted". - """ - config = AmazonConverseConfig() - - data = config._transform_request_helper( +@pytest.mark.parametrize( + "model", + [ + "anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8", + "amazon.nova-pro-v1:0", + "us.meta.llama4-maverick-17b-instruct-v1:0", + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abcdef123456", + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.amazon.nova-pro-v1:0", + ], +) +def test_client_metadata_stripped_from_converse_request(model): + data = AmazonConverseConfig()._transform_request_helper( model=model, system_content_blocks=[], optional_params={ @@ -999,71 +1002,11 @@ def test_client_metadata_stripped_for_anthropic_converse_request(model): messages=None, ) - fields = data.get("additionalModelRequestFields", {}) + fields = data["additionalModelRequestFields"] assert "client_metadata" not in fields assert fields["anthropic_beta"] == ["computer-use-2025-01-24"] -def test_client_metadata_kept_for_non_anthropic_converse_request(): - """Only Anthropic is known to reject ``client_metadata``, so other families keep the passthrough.""" - config = AmazonConverseConfig() - - data = config._transform_request_helper( - model="amazon.nova-pro-v1:0", - system_content_blocks=[], - optional_params={ - "maxTokens": 16, - "client_metadata": {"originator": "codex_cli_rs"}, - }, - messages=None, - ) - - assert data["additionalModelRequestFields"]["client_metadata"] == {"originator": "codex_cli_rs"} - - -@pytest.mark.parametrize( - "model", - [ - "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abcdef123456", - "arn:aws:bedrock:us-east-1:123456789012:provisioned-model/abcdef123456", - ], -) -def test_client_metadata_stripped_for_arn_models_converse(model): - """An ARN hides which family serves the request, and pointing one at Claude is how - teams route codex traffic, so the field has to go there too or the 400 comes back.""" - config = AmazonConverseConfig() - - data = config._transform_request_helper( - model=model, - system_content_blocks=[], - optional_params={ - "maxTokens": 16, - "client_metadata": {"originator": "codex_cli_rs"}, - }, - messages=None, - ) - - assert "client_metadata" not in data.get("additionalModelRequestFields", {}) - - -def test_client_metadata_kept_for_arn_naming_a_non_anthropic_family(): - """An inference profile ARN that still spells out the family is resolvable, so a - non-Anthropic one keeps its passthrough.""" - config = AmazonConverseConfig() - - data = config._transform_request_helper( - model="arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.amazon.nova-pro-v1:0", - system_content_blocks=[], - optional_params={ - "maxTokens": 16, - "client_metadata": {"originator": "codex_cli_rs"}, - }, - messages=None, - ) - - assert data["additionalModelRequestFields"]["client_metadata"] == {"originator": "codex_cli_rs"} - - def test_parallel_tool_calls_config_kept_for_sonnet_5(monkeypatch): old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost From 7a35c34303e944f68e69299346ec04371b5f59c7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:30:14 -0700 Subject: [PATCH 077/175] fix(models): add the us-gov. geo inference profile keys for Claude Sonnet 5 and Opus 4.8 --- ...odel_prices_and_context_window_backup.json | 64 +++++++++++++++++++ model_prices_and_context_window.json | 64 +++++++++++++++++++ .../test_bedrock_usgov_pricing.py | 19 ++++-- 3 files changed, 142 insertions(+), 5 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 30621a17df3..28c503966f3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -42008,6 +42008,70 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024 }, + "us-gov.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us-gov.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 30621a17df3..28c503966f3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -42008,6 +42008,70 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024 }, + "us-gov.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us-gov.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, diff --git a/tests/test_litellm/test_bedrock_usgov_pricing.py b/tests/test_litellm/test_bedrock_usgov_pricing.py index f9e8fd4c46c..f7d95ecda01 100644 --- a/tests/test_litellm/test_bedrock_usgov_pricing.py +++ b/tests/test_litellm/test_bedrock_usgov_pricing.py @@ -135,15 +135,24 @@ CLAUDE_GOV_EXPECTED = { } +USGOV_CLAUDE_KEY_TEMPLATES = { + "bedrock/us-gov-east-1/{base_key}": "bedrock", + "bedrock/us-gov-west-1/{base_key}": "bedrock", + "us-gov.{base_key}": "bedrock_converse", +} + + @pytest.mark.parametrize("base_key", CLAUDE_GOV_EXPECTED) -@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"]) -def test_usgov_claude_sonnet5_opus48_pricing(model_data, region, base_key): - """Sonnet 5 and Opus 4.8 gov entries must match the rates AWS publishes - for both GovCloud regions on the Bedrock pricing page (1.2x global). +@pytest.mark.parametrize("key_template,expected_provider", USGOV_CLAUDE_KEY_TEMPLATES.items()) +def test_usgov_claude_sonnet5_opus48_pricing(model_data, key_template, expected_provider, base_key): + """Sonnet 5 and Opus 4.8 gov entries, both in-region keys and the us-gov. + geo inference profile the model cards list for GovCloud, must match the + rates AWS publishes on the Bedrock pricing page (1.2x global). """ - gov_key = f"bedrock/{region}/{base_key}" + gov_key = key_template.format(base_key=base_key) assert gov_key in model_data, f"Missing model entry: {gov_key}" info = model_data[gov_key] + assert info["litellm_provider"] == expected_provider for field, expected in CLAUDE_GOV_EXPECTED[base_key].items(): assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})" ratio = info[field] / model_data[base_key][field] From 0c539614451869b790897c0576d202820d85ad59 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:44:51 -0700 Subject: [PATCH 078/175] fix(proxy-extras): give prisma migrate deploy its own timeout budget --- .../litellm_proxy_extras/prisma_toolchain.py | 25 +++++- .../litellm_proxy_extras/utils.py | 24 ++++-- .../test_prisma_toolchain.py | 78 ++++++++++++++++++- 3 files changed, 114 insertions(+), 13 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py b/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py index f3b55fd4d96..5feb7a953b4 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py +++ b/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py @@ -14,9 +14,19 @@ then fails on a Node binary that was never written. Deleting a cache directory that exists without a Node binary is what turns a killed bootstrap back into a recoverable one. -Both budgets are overridable so an operator can widen them without a release: -``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install and -``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every individual Prisma command. +``prisma migrate deploy`` is the other command whose runtime is not a +constant: it grows with the number of pending migrations, so a fresh database +that has to replay every migration this package ships overruns a per-command +budget sized for the short bookkeeping commands, on a laptop as much as on a +slow CI runner. The Python ``prisma`` wrapper spawns Node and the schema engine +as separate children, so killing the wrapper on timeout leaves them running: +the retry then contends with that orphan for Prisma's advisory lock and cannot +finish any sooner. Migrate deploy therefore runs under its own budget. + +All three budgets are overridable so an operator can widen them without a +release: ``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install, +``LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT`` for ``prisma migrate deploy`` and +``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every other Prisma command. """ import math @@ -36,10 +46,12 @@ except ImportError: PRISMA_COMMAND_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_COMMAND_TIMEOUT" PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_BOOTSTRAP_TIMEOUT" +PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT" NODEENV_CACHE_DIR_ENV_VAR = "PRISMA_NODEENV_CACHE_DIR" DEFAULT_PRISMA_COMMAND_TIMEOUT = 60.0 DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT = 600.0 +DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT = 600.0 BOOTSTRAP_ARG = "--version" @@ -88,6 +100,13 @@ def prisma_bootstrap_timeout() -> float: ) +def prisma_migrate_deploy_timeout() -> float: + """Seconds one ``prisma migrate deploy`` may run for, however many migrations are pending.""" + return _timeout_from_env( + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT + ) + + def nodeenv_cache_dir() -> Optional[Path]: """Where Prisma installs its private Node runtime, or None if unknowable.""" override = os.getenv(NODEENV_CACHE_DIR_ENV_VAR) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index b8032dd0d28..ab9ec1e8a3a 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -15,8 +15,10 @@ from litellm_proxy_extras.replica_identity import ( apply_replica_identity_full, ) from litellm_proxy_extras.prisma_toolchain import ( + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, ensure_prisma_toolchain, prisma_command_timeout, + prisma_migrate_deploy_timeout, ) @@ -698,12 +700,13 @@ class ProxyExtrasDBManager: original_dir = os.getcwd() os.chdir(migrations_dir) + deploy_timeout = prisma_migrate_deploy_timeout() try: for attempt in range(4): try: result = subprocess.run( [_get_prisma_command(), "migrate", "deploy"], - timeout=prisma_command_timeout(), + timeout=deploy_timeout, check=True, capture_output=True, text=True, @@ -713,8 +716,12 @@ class ProxyExtrasDBManager: return True except subprocess.TimeoutExpired: - logger.info( - f"prisma migrate deploy attempt {attempt + 1} timed out, retrying" + logger.warning( + "prisma migrate deploy attempt %s timed out after %ss, retrying. " + "Raise %s if this database needs longer to apply its pending migrations.", + attempt + 1, + deploy_timeout, + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, ) time.sleep(random.randrange(5, 15)) continue @@ -823,7 +830,8 @@ class ProxyExtrasDBManager: "Database migration failed after 4 attempts (retry loop " "exhausted by timeouts or repeated idempotent-recovery " "continues). Check database connectivity, load, and " - "_prisma_migrations ledger state." + "_prisma_migrations ledger state, and raise " + f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out." ) finally: os.chdir(original_dir) @@ -908,7 +916,7 @@ class ProxyExtrasDBManager: # Set migrations directory for Prisma result = subprocess.run( [_get_prisma_command(), "migrate", "deploy"], - timeout=prisma_command_timeout(), + timeout=prisma_migrate_deploy_timeout(), check=True, capture_output=True, text=True, @@ -1126,7 +1134,11 @@ class ProxyExtrasDBManager: ) return True except subprocess.TimeoutExpired: - logger.info(f"Attempt {attempt + 1} timed out") + logger.warning( + "Attempt %s timed out. Raise %s if this database needs longer to apply its pending migrations.", + attempt + 1, + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, + ) time.sleep(random.randrange(5, 15)) except subprocess.CalledProcessError as e: attempts_left = 3 - attempt diff --git a/tests/proxy_migration_tests/test_prisma_toolchain.py b/tests/proxy_migration_tests/test_prisma_toolchain.py index d12a2c4dd4e..1bd440d37b4 100644 --- a/tests/proxy_migration_tests/test_prisma_toolchain.py +++ b/tests/proxy_migration_tests/test_prisma_toolchain.py @@ -7,6 +7,11 @@ attempt fails identically. These tests pin the two behaviours that keep a container recoverable: an incomplete cache is deleted before Prisma is invoked, and the install gets a budget of its own rather than sharing the one that bounds each migration command. + +``prisma migrate deploy`` gets a budget of its own for the same reason: its +runtime grows with the number of pending migrations, so a fresh database that +replays every migration overran the per-command budget on slow machines and +the proxy gave up after four identical timeouts. """ import ast @@ -14,19 +19,23 @@ import json import os import sys import time +from collections.abc import Callable from pathlib import Path import pytest from litellm_proxy_extras.prisma_toolchain import ( DEFAULT_PRISMA_COMMAND_TIMEOUT, + DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT, PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR, PRISMA_COMMAND_TIMEOUT_ENV_VAR, + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, ensure_prisma_toolchain, heal_incomplete_nodeenv_cache, node_binary_path, prisma_bootstrap_timeout, prisma_command_timeout, + prisma_migrate_deploy_timeout, ) from litellm_proxy_extras.utils import ProxyExtrasDBManager @@ -42,13 +51,24 @@ import time args = sys.argv[1:] cache_dir = os.environ["PRISMA_NODEENV_CACHE_DIR"] -with pathlib.Path(os.environ["FAKE_PRISMA_LOG"]).open("a") as log: +log_path = pathlib.Path(os.environ["FAKE_PRISMA_LOG"]) +earlier_deploys = sum( + 1 + for line in (log_path.read_text().splitlines() if log_path.exists() else []) + if json.loads(line)["args"][:2] == ["migrate", "deploy"] +) +with log_path.open("a") as log: log.write( json.dumps({{"args": args, "cache_dir_present": os.path.isdir(cache_dir)}}) + "\\n" ) time.sleep(float(os.environ.get("FAKE_PRISMA_SLEEP", "0"))) if args[:2] == ["migrate", "deploy"]: + if earlier_deploys == 0: + time.sleep(float(os.environ.get("FAKE_PRISMA_FIRST_DEPLOY_SLEEP", "0"))) + elif os.environ.get("FAKE_PRISMA_LATER_DEPLOY_STDERR"): + print(os.environ["FAKE_PRISMA_LATER_DEPLOY_STDERR"], file=sys.stderr) + sys.exit(1) print("No pending migrations to apply") sys.exit(0) """ @@ -80,9 +100,14 @@ def toolchain_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}") monkeypatch.delenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, raising=False) monkeypatch.delenv(PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR, raising=False) + monkeypatch.delenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, raising=False) return cache_dir, log_path +def _deploy_calls(log_path: Path) -> list[list[str]]: + return [call["args"] for call in _fake_prisma_calls(log_path) if call["args"][:2] == ["migrate", "deploy"]] + + def _make_incomplete_cache(cache_dir: Path) -> None: (cache_dir / "lib").mkdir(parents=True) (cache_dir / "bin").mkdir() @@ -209,25 +234,70 @@ def test_setup_database_prepares_the_toolchain_before_migrating( assert calls[0]["cache_dir_present"] is False +@pytest.mark.parametrize("use_v2_resolver", [False, True], ids=["v1", "v2"]) +def test_migrate_deploy_is_not_bounded_by_the_per_command_timeout( + toolchain_env: tuple[Path, Path], + monkeypatch: pytest.MonkeyPatch, + use_v2_resolver: bool, +) -> None: + """A fresh database replays every migration, which takes longer than any bookkeeping command.""" + _, log_path = toolchain_env + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, "1") + monkeypatch.setenv("FAKE_PRISMA_FIRST_DEPLOY_SLEEP", "3") + + assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True + assert _deploy_calls(log_path) == [["migrate", "deploy"]] + + +def test_migrate_deploy_stops_at_its_own_timeout( + toolchain_env: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + """The deploy budget still bounds a deploy that hangs, so boot cannot wait forever.""" + _, log_path = toolchain_env + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, "1") + monkeypatch.setenv("FAKE_PRISMA_FIRST_DEPLOY_SLEEP", "60") + monkeypatch.setenv("FAKE_PRISMA_LATER_DEPLOY_STDERR", "Error: P3018 permission denied for schema public") + + started = time.monotonic() + with pytest.raises(RuntimeError, match="insufficient permissions"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + elapsed = time.monotonic() - started + + assert len(_deploy_calls(log_path)) == 2 + assert elapsed < 30 + + @pytest.mark.parametrize( "raw", ["", "0", "-5", "not-a-number", "nan", "inf", "-inf", "1e400"], ) +@pytest.mark.parametrize( + ("env_var", "read_timeout", "default"), + [ + (PRISMA_COMMAND_TIMEOUT_ENV_VAR, prisma_command_timeout, DEFAULT_PRISMA_COMMAND_TIMEOUT), + (PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, prisma_migrate_deploy_timeout, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT), + ], + ids=["command", "migrate_deploy"], +) def test_unusable_timeout_override_falls_back_to_the_default( - raw: str, monkeypatch: pytest.MonkeyPatch + raw: str, env_var: str, read_timeout: Callable[[], float], default: float, monkeypatch: pytest.MonkeyPatch ) -> None: """A non-finite override would silently disable the timeout it configures.""" - monkeypatch.setenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, raw) + monkeypatch.setenv(env_var, raw) - assert prisma_command_timeout() == DEFAULT_PRISMA_COMMAND_TIMEOUT + assert read_timeout() == default def test_timeout_overrides_are_independent(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, "12") monkeypatch.setenv(PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR, "900") + monkeypatch.setenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, "1200") assert prisma_command_timeout() == 12 assert prisma_bootstrap_timeout() == 900 + assert prisma_migrate_deploy_timeout() == 1200 @pytest.mark.parametrize("module", ["utils.py", "replica_identity.py"]) From ffc0a8e428a4d8af7e5b130bddb4f0f9cf0cb229 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:51:17 -0700 Subject: [PATCH 079/175] fix: run access group key sync UPDATEs on the writer, not the read replica (#39128) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../access_group_key_sync.py | 6 +- .../test_access_group_key_sync.py | 57 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/proxy/management_helpers/test_access_group_key_sync.py diff --git a/litellm/proxy/management_helpers/access_group_key_sync.py b/litellm/proxy/management_helpers/access_group_key_sync.py index 5d43cb29978..c9f93fae0d9 100644 --- a/litellm/proxy/management_helpers/access_group_key_sync.py +++ b/litellm/proxy/management_helpers/access_group_key_sync.py @@ -38,6 +38,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_checks import ( _delete_cache_access_object, # pyright: ignore[reportPrivateUsage] # the access-group endpoints reach for this same cache primitive ) +from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient from litellm.repositories.table_repositories import AccessGroupRepository @@ -72,8 +73,9 @@ _REPOINT_KEY_SQL: Final = ( def _raw_executor(prisma_client: object) -> _RawExecutor: - """Narrow the untyped Prisma client down to the raw-query call this module makes.""" - return AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client + """Narrow the untyped Prisma client down to the raw-query call this module makes, pinned to the writer.""" + db: Final = AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client + return WriterPinnedClient(db).db # pyright: ignore[reportAny, reportReturnType] # untyped Prisma client behind the pin async def _invalidate_access_group_cache(access_group_id: str) -> None: diff --git a/tests/test_litellm/proxy/management_helpers/test_access_group_key_sync.py b/tests/test_litellm/proxy/management_helpers/test_access_group_key_sync.py new file mode 100644 index 00000000000..60c36e33e09 --- /dev/null +++ b/tests/test_litellm/proxy/management_helpers/test_access_group_key_sync.py @@ -0,0 +1,57 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.proxy.db.prisma_client import PrismaWrapper +from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper +from litellm.proxy.management_helpers.access_group_key_sync import ( + sync_key_access_group_membership, + sync_key_regeneration_access_group_membership, +) + + +def _routed_prisma_client(): + writer_inner = MagicMock(name="writer_prisma") + reader_inner = MagicMock(name="reader_prisma") + writer_inner.query_raw = AsyncMock(return_value=[]) + reader_inner.query_raw = AsyncMock(return_value=[]) + writer = PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False) + reader = PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False) + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + return SimpleNamespace(db=routing), writer_inner, reader_inner + + +@pytest.mark.asyncio +async def test_regeneration_repoint_update_runs_on_the_writer(): + prisma_client, writer_inner, reader_inner = _routed_prisma_client() + + await sync_key_regeneration_access_group_membership( + prisma_client=prisma_client, + previous_key_token="old-token", + new_key_token="new-token", + data=None, + existing_key_row=MagicMock(), + ) + + writer_inner.query_raw.assert_awaited_once() + assert writer_inner.query_raw.await_args.args[0].startswith('UPDATE "LiteLLM_AccessGroupTable"') + reader_inner.query_raw.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_membership_attach_and_detach_updates_run_on_the_writer(): + prisma_client, writer_inner, reader_inner = _routed_prisma_client() + + await sync_key_access_group_membership( + prisma_client=prisma_client, + key_token="token", + previous_access_group_ids=["ag-old"], + updated_access_group_ids=["ag-new"], + ) + + assert writer_inner.query_raw.await_count == 2 + assert all( + call.args[0].startswith('UPDATE "LiteLLM_AccessGroupTable"') for call in writer_inner.query_raw.await_args_list + ) + reader_inner.query_raw.assert_not_awaited() From a9d3a0746c582de8910d0a7078489ac961e436e8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:59:11 -0700 Subject: [PATCH 080/175] fix(models): price Azure DeepSeek V4 Flash 0731 from its own meters under the catalog id --- ...odel_prices_and_context_window_backup.json | 22 +++---------------- model_prices_and_context_window.json | 22 +++---------------- 2 files changed, 6 insertions(+), 38 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 893ac49f5c7..0af5a89742e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -10172,31 +10172,15 @@ "supports_tool_choice": true }, "azure_ai/DeepSeek-V4-Flash-0731": { - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, + "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.1e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "azure_ai/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 2.8e-08, - "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 5.1e-07, + "output_cost_per_token": 1.32e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_prompt_caching": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 893ac49f5c7..0af5a89742e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -10172,31 +10172,15 @@ "supports_tool_choice": true }, "azure_ai/DeepSeek-V4-Flash-0731": { - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, + "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.1e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "azure_ai/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 2.8e-08, - "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 5.1e-07, + "output_cost_per_token": 1.32e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_prompt_caching": true, From e0be9a35e6838505b5a2ae2ecf93c01579a02f56 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:02:43 -0700 Subject: [PATCH 081/175] fix(deps): raise the pypdf floor to 6.16.1 for three new advisories GHSA-jp53-mhqp-8xcg (fixed in 6.16.0), GHSA-23w6-3w8w-8484 and GHSA-763m-79hh-57f2 (fixed in 6.16.1) flag pypdf 6.15.0 in uv.lock and keep osv-scan red alongside the tornado advisories. The proxy-runtime extra now requires pypdf>=6.16.1 and the lock resolves 6.16.2. --- pyproject.toml | 2 +- uv.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d0f14722acd..60162544612 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,7 +161,7 @@ proxy-runtime = [ "mangum>=0.17.0,<1.0", "azure-ai-contentsafety>=1.0.0,<2.0", "azure-storage-file-datalake>=12.20.0,<13.0", - "pypdf>=6.12.0,<7.0", + "pypdf>=6.16.1,<7.0", "llm-sandbox>=0.3.39,<1.0", "detect-secrets>=1.5.0,<2.0", ] diff --git a/uv.lock b/uv.lock index 8bac024d49e..aa59ff7b229 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-29T20:52:40.322465Z" +exclude-newer = "2026-08-30T17:51:25.171404Z" exclude-newer-span = "P3D" [manifest] @@ -4552,7 +4552,7 @@ requires-dist = [ { name = "pydantic-settings", specifier = ">=2.14.1,<3.0" }, { name = "pyjwt", marker = "extra == 'proxy'", specifier = ">=2.13.0,<3.0" }, { name = "pynacl", marker = "extra == 'proxy'", specifier = ">=1.6.2,<2.0" }, - { name = "pypdf", marker = "extra == 'proxy-runtime'", specifier = ">=6.12.0,<7.0" }, + { name = "pypdf", marker = "extra == 'proxy-runtime'", specifier = ">=6.16.1,<7.0" }, { name = "pyroscope-io", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.8.16,<1.0" }, { name = "python-dotenv", specifier = ">=1.0.0,<2.0" }, { name = "python-multipart", marker = "extra == 'proxy'", specifier = ">=0.0.27,<1.0" }, @@ -7564,14 +7564,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.15.0" +version = "6.16.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/17/ee75a92718ec7212de831e71454d702225aa5e474a805cce169806044453/pypdf-6.15.0.tar.gz", hash = "sha256:d39c4d955a76409284a905e2d65b40076d77ab76129e0faaeeb6612403ecfc79", size = 6993794, upload-time = "2026-08-06T13:06:49.929Z" } +sdist = { url = "https://files.pythonhosted.org/packages/44/66/54212e75406afd9f3e933d0dda23072f6aecc55c5a273077dc2e0b028b23/pypdf-6.16.2.tar.gz", hash = "sha256:595647f6191de6f402cfde1d0c455d6cbccbd509aac32b34783009c032de5d6e", size = 7008996, upload-time = "2026-08-23T13:50:07.135Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/72/ce3067ac31e214a66388159f8462ddb8c13dd00170f24d555a1f1ae8ee91/pypdf-6.15.0-py3-none-any.whl", hash = "sha256:14e001d6504822cb1ca9c7ed9a69bccb320f59b320730f55af804361abe4d5ee", size = 378123, upload-time = "2026-08-06T13:06:47.709Z" }, + { url = "https://files.pythonhosted.org/packages/13/f1/a2da3b55acd4ab737bf728c97edaaed5ec1d3c1236acb639dcdfa97e42c7/pypdf-6.16.2-py3-none-any.whl", hash = "sha256:c8b09a59399062fb45a1b8156c18a787a10a3dae03ac9674397a226712c94604", size = 385060, upload-time = "2026-08-23T13:50:05.349Z" }, ] [[package]] From dbc126cfc97734e47066ab988c3ac1090e7f830a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:07:30 -0700 Subject: [PATCH 082/175] fix(hosted_vllm): forward truncate_prompt_tokens on rerank requests --- .../llms/hosted_vllm/rerank/transformation.py | 21 ++- litellm/types/rerank.py | 21 ++- .../test_hosted_vllm_rerank_transformation.py | 122 +++++++++++++++++- 3 files changed, 154 insertions(+), 10 deletions(-) diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 0e8fa294f5d..265eb350fc6 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -3,6 +3,7 @@ Transformation logic for Hosted VLLM rerank """ from collections.abc import Mapping +from types import MappingProxyType from typing import Any, Final import httpx @@ -13,6 +14,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.secret_managers.main import get_secret_str from litellm.types.rerank import ( + HostedVLLMRerankTruncationParams, OptionalRerankParams, RerankBilledUnits, RerankRequest, @@ -62,7 +64,11 @@ class HostedVLLMRerankConfig(BaseRerankConfig): "top_n", "rank_fields", "return_documents", + "max_tokens_per_doc", "instruction", + "truncate_prompt_tokens", + "truncation_side", + "max_tokens_per_query", ] def map_cohere_rerank_params( @@ -100,7 +106,15 @@ class HostedVLLMRerankConfig(BaseRerankConfig): if instruction is not None: mapped_params["instruction"] = instruction - return dict(mapped_params) + truncation: Final = HostedVLLMRerankTruncationParams.model_validate(non_default_params or MappingProxyType({})) + forwarded: Final[OptionalRerankParams] = { + **mapped_params, + "max_tokens_per_doc": max_tokens_per_doc, + "truncate_prompt_tokens": truncation.truncate_prompt_tokens, + "truncation_side": truncation.truncation_side, + "max_tokens_per_query": truncation.max_tokens_per_query, + } + return dict(forwarded) def validate_environment( self, @@ -138,6 +152,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): if "documents" not in optional_rerank_params: raise ValueError("documents is required for Hosted VLLM rerank") + truncation: Final = HostedVLLMRerankTruncationParams.model_validate(optional_rerank_params) rerank_request: Final = RerankRequest( model=model, query=optional_rerank_params["query"], @@ -146,6 +161,10 @@ class HostedVLLMRerankConfig(BaseRerankConfig): rank_fields=optional_rerank_params.get("rank_fields", None), return_documents=optional_rerank_params.get("return_documents", None), instruction=optional_rerank_params.get("instruction", None), + max_tokens_per_doc=truncation.max_tokens_per_doc, + truncate_prompt_tokens=truncation.truncate_prompt_tokens, + truncation_side=truncation.truncation_side, + max_tokens_per_query=truncation.max_tokens_per_query, ) return rerank_request.model_dump(exclude_none=True) diff --git a/litellm/types/rerank.py b/litellm/types/rerank.py index 903781b2ccd..a76e6cf1187 100644 --- a/litellm/types/rerank.py +++ b/litellm/types/rerank.py @@ -4,8 +4,10 @@ https://docs.cohere.com/reference/rerank """ -from pydantic import BaseModel, PrivateAttr -from typing_extensions import Required, TypedDict +from typing import Literal + +from pydantic import BaseModel, ConfigDict, PrivateAttr +from typing_extensions import ReadOnly, Required, TypedDict class RerankRequest(BaseModel): @@ -21,6 +23,18 @@ class RerankRequest(BaseModel): # (e.g. hosted vLLM / Qwen3-Reranker, DeepInfra). Omitted from the outgoing # request when None, so this is fully backward-compatible. instruction: str | None = None + truncate_prompt_tokens: int | None = None + truncation_side: Literal["left", "right"] | None = None + max_tokens_per_query: int | None = None + + +class HostedVLLMRerankTruncationParams(BaseModel): + model_config = ConfigDict(frozen=True) + + truncate_prompt_tokens: int | None = None + truncation_side: Literal["left", "right"] | None = None + max_tokens_per_query: int | None = None + max_tokens_per_doc: int | None = None class OptionalRerankParams(TypedDict, total=False): @@ -32,6 +46,9 @@ class OptionalRerankParams(TypedDict, total=False): max_chunks_per_doc: int | None max_tokens_per_doc: int | None instruction: str | None + truncate_prompt_tokens: ReadOnly[int | None] + truncation_side: ReadOnly[Literal["left", "right"] | None] + max_tokens_per_query: ReadOnly[int | None] class RerankBilledUnits(TypedDict, total=False): diff --git a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py index e6e6aa946d5..da27ea1ac58 100644 --- a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py @@ -1,8 +1,13 @@ +import json import os import sys +from unittest.mock import MagicMock, patch +import httpx import pytest +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig from litellm.rerank_api.rerank_utils import get_optional_rerank_params from litellm.types.rerank import ( @@ -87,9 +92,7 @@ class TestHostedVLLMRerankTransform: assert "instruction" not in body def test_map_cohere_rerank_params_raises_on_max_chunks_per_doc(self): - with pytest.raises( - ValueError, match="Hosted VLLM does not support max_chunks_per_doc" - ): + with pytest.raises(ValueError, match="Hosted VLLM does not support max_chunks_per_doc"): self.config.map_cohere_rerank_params( non_default_params=None, model=self.model, @@ -104,12 +107,10 @@ class TestHostedVLLMRerankTransform: url = self.config.get_complete_url(base, self.model) assert url == "https://api.example.com/rerank" # Already ends with /rerank - url2 = self.config.get_complete_url( - "https://api.example.com/rerank", self.model - ) + url2 = self.config.get_complete_url("https://api.example.com/rerank", self.model) assert url2 == "https://api.example.com/rerank" # Raises if api_base is None - with pytest.raises(ValueError, match='api_base must be provided for Hosted VLLM rerank'): + with pytest.raises(ValueError, match="api_base must be provided for Hosted VLLM rerank"): self.config.get_complete_url(None, self.model) def test_transform_response(self): @@ -173,3 +174,110 @@ class TestGetOptionalRerankParamsInstruction: documents=["doc1", "doc2"], ) assert "instruction" not in params + + +class TestHostedVLLMRerankTruncationParams: + def setup_method(self): + self.config = HostedVLLMRerankConfig() + self.model = "hosted-vllm-model" + + def test_map_cohere_rerank_params_forwards_vllm_truncation_params(self): + params = self.config.map_cohere_rerank_params( + non_default_params={ + "truncate_prompt_tokens": 512, + "truncation_side": "left", + "max_tokens_per_query": 64, + "metadata": {"user_api_key": "sk-test"}, + }, + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + max_tokens_per_doc=128, + ) + assert params["truncate_prompt_tokens"] == 512 + assert params["truncation_side"] == "left" + assert params["max_tokens_per_query"] == 64 + assert params["max_tokens_per_doc"] == 128 + assert "metadata" not in params + + def test_map_cohere_rerank_params_omits_truncation_params_when_absent(self): + params = self.config.map_cohere_rerank_params( + non_default_params={"metadata": {"user_api_key": "sk-test"}}, + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + ) + body = self.config.transform_rerank_request(model=self.model, optional_rerank_params=params, headers={}) + truncation_keys = {"truncate_prompt_tokens", "truncation_side", "max_tokens_per_query", "max_tokens_per_doc"} + assert not truncation_keys & body.keys() + assert body == { + "model": self.model, + "query": "test query", + "documents": ["doc1", "doc2"], + "return_documents": True, + } + + def test_map_cohere_rerank_params_rejects_invalid_truncation_side(self): + with pytest.raises(ValueError, match="truncation_side"): + self.config.map_cohere_rerank_params( + non_default_params={"truncation_side": "middle"}, + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + ) + + def test_transform_request_forwards_truncation_params(self): + body = self.config.transform_rerank_request( + model=self.model, + optional_rerank_params={ + "query": "test query", + "documents": ["doc1", "doc2"], + "truncate_prompt_tokens": 512, + "truncation_side": "left", + "max_tokens_per_query": 64, + "max_tokens_per_doc": 128, + }, + headers={}, + ) + assert body["truncate_prompt_tokens"] == 512 + assert body["truncation_side"] == "left" + assert body["max_tokens_per_query"] == 64 + assert body["max_tokens_per_doc"] == 128 + + def test_transform_request_omits_truncation_params_when_absent(self): + body = self.config.transform_rerank_request( + model=self.model, + optional_rerank_params={"query": "test query", "documents": ["doc1", "doc2"]}, + headers={}, + ) + assert "truncate_prompt_tokens" not in body + assert "truncation_side" not in body + assert "max_tokens_per_query" not in body + assert "max_tokens_per_doc" not in body + + def test_rerank_sends_truncate_prompt_tokens_to_vllm(self): + client = HTTPHandler() + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "id": "score-1", + "results": [{"index": 0, "relevance_score": 0.5}], + "usage": {"total_tokens": 512}, + } + with patch.object(client, "post", return_value=mock_response) as mock_post: + litellm.rerank( + model="hosted_vllm/BAAI/bge-reranker-base", + api_base="http://vllm.local:8000", + query="List all the unique case ids", + documents=["a document longer than the reranker context window"], + truncate_prompt_tokens=512, + truncation_side="left", + client=client, + ) + sent_body = json.loads(mock_post.call_args.kwargs["data"]) + assert mock_post.call_args.kwargs["url"] == "http://vllm.local:8000/rerank" + assert sent_body["truncate_prompt_tokens"] == 512 + assert sent_body["truncation_side"] == "left" From dfaf23523453de12278e3d30801075c4da6ee903 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:08:45 -0700 Subject: [PATCH 083/175] fix(bedrock): honor BEDROCK_MANTLE_API_BASE on bedrock/mantle messages and chat URLs --- litellm/llms/bedrock/common_utils.py | 7 ++-- .../test_litellm/llms/bedrock/test_mantle.py | 42 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 66ee5f10679..048d023a1bd 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -758,12 +758,13 @@ def build_mantle_messages_url( """Build the bedrock-mantle Anthropic /messages URL. Honors an explicit endpoint override (``api_base``, then - ``aws_bedrock_runtime_endpoint``) so private VPC / VPCE / GovCloud Mantle - endpoints are reachable; otherwise falls back to the public regional host. + ``aws_bedrock_runtime_endpoint``, then ``BEDROCK_MANTLE_API_BASE``) so + private VPC / VPCE / GovCloud Mantle endpoints are reachable; otherwise + falls back to the public regional host. The mantle messages path is appended unless the override already carries it, so callers can pass either the host or the full messages URL. """ - override: Final = api_base or aws_bedrock_runtime_endpoint + override: Final = api_base or aws_bedrock_runtime_endpoint or get_secret_str("BEDROCK_MANTLE_API_BASE") if override: base: Final = override.rstrip("/") if base.endswith(MANTLE_MESSAGES_PATH): diff --git a/tests/test_litellm/llms/bedrock/test_mantle.py b/tests/test_litellm/llms/bedrock/test_mantle.py index d34517f61f6..d1d1ba447fb 100644 --- a/tests/test_litellm/llms/bedrock/test_mantle.py +++ b/tests/test_litellm/llms/bedrock/test_mantle.py @@ -128,6 +128,12 @@ def test_mantle_messages_url_construction(): _VPC_ENDPOINT = "https://vpce-0a1b2c3d.bedrock-mantle.us-gov-west-1.vpce.amazonaws.com" +@pytest.fixture(autouse=True) +def no_ambient_mantle_api_base(monkeypatch): + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + + + def test_mantle_chat_url_honors_api_base_host(): config = AmazonMantleConfig() url = config.get_complete_url( @@ -193,6 +199,42 @@ def test_mantle_messages_url_honors_aws_bedrock_runtime_endpoint(): assert url == f"{_VPC_ENDPOINT}/anthropic/v1/messages" +_ENV_ENDPOINT = "https://bedrock-mantle.us-east-1.api.aws.internal.example.com" + + +@pytest.mark.parametrize("config_cls", [AmazonMantleConfig, AmazonMantleMessagesConfig]) +def test_mantle_url_honors_bedrock_mantle_api_base_env(monkeypatch, config_cls): + monkeypatch.setenv("BEDROCK_MANTLE_API_BASE", _ENV_ENDPOINT) + url = config_cls().get_complete_url( + api_base=None, + api_key=None, + model="mantle/anthropic.claude-mythos-preview", + optional_params={"aws_region_name": "us-east-1"}, + litellm_params={}, + ) + assert url == f"{_ENV_ENDPOINT}/anthropic/v1/messages" + + +@pytest.mark.parametrize("config_cls", [AmazonMantleConfig, AmazonMantleMessagesConfig]) +@pytest.mark.parametrize( + ("api_base", "optional_params"), + [ + (_VPC_ENDPOINT, {"aws_region_name": "us-gov-west-1"}), + (None, {"aws_region_name": "us-gov-west-1", "aws_bedrock_runtime_endpoint": _VPC_ENDPOINT}), + ], +) +def test_mantle_url_explicit_endpoint_beats_bedrock_mantle_api_base_env(monkeypatch, config_cls, api_base, optional_params): + monkeypatch.setenv("BEDROCK_MANTLE_API_BASE", _ENV_ENDPOINT) + url = config_cls().get_complete_url( + api_base=api_base, + api_key=None, + model="mantle/anthropic.claude-mythos-preview", + optional_params=optional_params, + litellm_params={}, + ) + assert url == f"{_VPC_ENDPOINT}/anthropic/v1/messages" + + def test_mantle_transform_request_strips_prefix_and_adds_model(): config = AmazonMantleConfig() request = config.transform_request( From 8d00220acef335234797b00cf3b1f1ee73db2b3a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:19:20 -0700 Subject: [PATCH 084/175] test(hosted_vllm): annotate rerank truncation test locals as Final --- .../test_hosted_vllm_rerank_transformation.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py index da27ea1ac58..49d58cc28c4 100644 --- a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py @@ -1,6 +1,7 @@ import json import os import sys +from typing import Final from unittest.mock import MagicMock, patch import httpx @@ -182,7 +183,7 @@ class TestHostedVLLMRerankTruncationParams: self.model = "hosted-vllm-model" def test_map_cohere_rerank_params_forwards_vllm_truncation_params(self): - params = self.config.map_cohere_rerank_params( + params: Final = self.config.map_cohere_rerank_params( non_default_params={ "truncate_prompt_tokens": 512, "truncation_side": "left", @@ -202,15 +203,20 @@ class TestHostedVLLMRerankTruncationParams: assert "metadata" not in params def test_map_cohere_rerank_params_omits_truncation_params_when_absent(self): - params = self.config.map_cohere_rerank_params( + params: Final = self.config.map_cohere_rerank_params( non_default_params={"metadata": {"user_api_key": "sk-test"}}, model=self.model, drop_params=False, query="test query", documents=["doc1", "doc2"], ) - body = self.config.transform_rerank_request(model=self.model, optional_rerank_params=params, headers={}) - truncation_keys = {"truncate_prompt_tokens", "truncation_side", "max_tokens_per_query", "max_tokens_per_doc"} + body: Final = self.config.transform_rerank_request(model=self.model, optional_rerank_params=params, headers={}) + truncation_keys: Final = { + "truncate_prompt_tokens", + "truncation_side", + "max_tokens_per_query", + "max_tokens_per_doc", + } assert not truncation_keys & body.keys() assert body == { "model": self.model, @@ -230,7 +236,7 @@ class TestHostedVLLMRerankTruncationParams: ) def test_transform_request_forwards_truncation_params(self): - body = self.config.transform_rerank_request( + body: Final = self.config.transform_rerank_request( model=self.model, optional_rerank_params={ "query": "test query", @@ -248,7 +254,7 @@ class TestHostedVLLMRerankTruncationParams: assert body["max_tokens_per_doc"] == 128 def test_transform_request_omits_truncation_params_when_absent(self): - body = self.config.transform_rerank_request( + body: Final = self.config.transform_rerank_request( model=self.model, optional_rerank_params={"query": "test query", "documents": ["doc1", "doc2"]}, headers={}, @@ -259,8 +265,8 @@ class TestHostedVLLMRerankTruncationParams: assert "max_tokens_per_doc" not in body def test_rerank_sends_truncate_prompt_tokens_to_vllm(self): - client = HTTPHandler() - mock_response = MagicMock(spec=httpx.Response) + client: Final = HTTPHandler() + mock_response: Final = MagicMock(spec=httpx.Response) mock_response.status_code = 200 mock_response.json.return_value = { "id": "score-1", @@ -277,7 +283,7 @@ class TestHostedVLLMRerankTruncationParams: truncation_side="left", client=client, ) - sent_body = json.loads(mock_post.call_args.kwargs["data"]) + sent_body: Final = json.loads(mock_post.call_args.kwargs["data"]) assert mock_post.call_args.kwargs["url"] == "http://vllm.local:8000/rerank" assert sent_body["truncate_prompt_tokens"] == 512 assert sent_body["truncation_side"] == "left" From ef14bed0296b4a6c48777b8f9df8018b11179c4b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:31:19 -0700 Subject: [PATCH 085/175] fix(hosted_vllm): reject invalid rerank truncation params with a 400 --- .../llms/hosted_vllm/rerank/transformation.py | 11 +++++++- .../test_hosted_vllm_rerank_transformation.py | 26 ++++++++++++------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 265eb350fc6..764d80c6f82 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -7,8 +7,10 @@ from types import MappingProxyType from typing import Any, Final import httpx +from pydantic import ValidationError from litellm._uuid import uuid +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig @@ -36,6 +38,13 @@ class HostedVLLMRerankError(BaseLLMException): super().__init__(status_code=status_code, message=message, headers=headers) +def validated_truncation_params(non_default_params: Mapping[str, object] | None) -> HostedVLLMRerankTruncationParams: + try: + return HostedVLLMRerankTruncationParams.model_validate(non_default_params or MappingProxyType({})) + except ValidationError as error: + raise UnsupportedParamsError(status_code=400, message=f"hosted_vllm rerank: {error}") from error + + class HostedVLLMRerankConfig(BaseRerankConfig): def __init__(self) -> None: pass @@ -106,7 +115,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): if instruction is not None: mapped_params["instruction"] = instruction - truncation: Final = HostedVLLMRerankTruncationParams.model_validate(non_default_params or MappingProxyType({})) + truncation: Final = validated_truncation_params(non_default_params) forwarded: Final[OptionalRerankParams] = { **mapped_params, "max_tokens_per_doc": max_tokens_per_doc, diff --git a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py index 49d58cc28c4..9a62fcf6f0f 100644 --- a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py @@ -202,6 +202,22 @@ class TestHostedVLLMRerankTruncationParams: assert params["max_tokens_per_doc"] == 128 assert "metadata" not in params + @pytest.mark.parametrize( + "bad_params", + [{"truncation_side": "middle"}, {"truncate_prompt_tokens": "lots"}, {"max_tokens_per_query": -1.5}], + ) + def test_map_cohere_rerank_params_rejects_invalid_truncation_params_as_400(self, bad_params: dict[str, object]): + with pytest.raises(litellm.UnsupportedParamsError) as raised: + self.config.map_cohere_rerank_params( + non_default_params=dict(bad_params), + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + ) + assert raised.value.status_code == 400 + assert next(iter(bad_params)) in str(raised.value) + def test_map_cohere_rerank_params_omits_truncation_params_when_absent(self): params: Final = self.config.map_cohere_rerank_params( non_default_params={"metadata": {"user_api_key": "sk-test"}}, @@ -225,16 +241,6 @@ class TestHostedVLLMRerankTruncationParams: "return_documents": True, } - def test_map_cohere_rerank_params_rejects_invalid_truncation_side(self): - with pytest.raises(ValueError, match="truncation_side"): - self.config.map_cohere_rerank_params( - non_default_params={"truncation_side": "middle"}, - model=self.model, - drop_params=False, - query="test query", - documents=["doc1", "doc2"], - ) - def test_transform_request_forwards_truncation_params(self): body: Final = self.config.transform_rerank_request( model=self.model, From d7ee215c57af44219d5a19f5042343ce98e9de2a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:32:04 -0700 Subject: [PATCH 086/175] fix(responses): keep namespace tools intact when a guardrail returns them unchanged Any pre_call guardrail on /v1/responses flattened Codex namespace tools into ns__member functions and wrote the flattened list back to the request, so the model called mcp__server__tool with no namespace and Codex rejected the call as unsupported. The handler now keeps the client's original tools, hands the guardrail a deep copy of the flattened ones, and rebuilds data["tools"] by matching the guardrail's output to the originals by type and name. Unchanged tools go back as the original objects, a dropped or edited namespace member changes only that member, and tools the guardrail injects are still appended. Fixes #39183 --- basedpyright-code-budget.json | 12 +- .../guardrail_translation/handler.py | 114 +++------- .../guardrail_translation/tool_merge.py | 177 +++++++++++++++ .../transformation.py | 161 ++++++++------ ruff-strict-budget.json | 2 +- ...test_openai_responses_guardrail_handler.py | 206 +++++++++++++++++- ...t_openai_responses_guardrail_tool_merge.py | 144 ++++++++++++ .../test_litellm_completion_responses.py | 13 ++ type-discipline-budget.json | 10 +- 9 files changed, 669 insertions(+), 170 deletions(-) create mode 100644 litellm/llms/openai/responses/guardrail_translation/tool_merge.py create mode 100644 tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index da788bf1ce3..e7a069de29a 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -3,7 +3,7 @@ "limit": 14076 }, "reportArgumentType": { - "limit": 2216 + "limit": 2215 }, "reportAssignmentType": { "limit": 319 @@ -24,7 +24,7 @@ "limit": 19 }, "reportExplicitAny": { - "limit": 4128 + "limit": 4127 }, "reportFunctionMemberAccess": { "limit": 7 @@ -99,7 +99,7 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 44364 + "limit": 44362 }, "reportUnknownLambdaType": { "limit": 109 @@ -117,13 +117,13 @@ "limit": 111 }, "reportUnnecessaryComparison": { - "limit": 692 + "limit": 687 }, "reportUnnecessaryContains": { - "limit": 5 + "limit": 4 }, "reportUnnecessaryIsInstance": { - "limit": 826 + "limit": 823 }, "reportUntypedBaseClass": { "limit": 0 diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 1530c154e93..5a5970fb867 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -28,6 +28,7 @@ Output: response.output is List[GenericResponseOutputItem] where each has: - text: str """ +import copy import time import uuid from collections.abc import Mapping, Sequence @@ -36,7 +37,6 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall -from openai.types.responses.tool_param import FunctionToolParam from pydantic import BaseModel, TypeAdapter from typing_extensions import ReadOnly, TypedDict @@ -49,6 +49,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_responses_stream_usage, stream_item_field, ) +from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -62,7 +63,6 @@ from litellm.types.llms.openai import ( ContentPartDonePartOutputText, ErrorEvent, ErrorEventError, - OpenAIMcpServerTool, OutputItemAddedEvent, OutputItemDoneEvent, OutputTextDeltaEvent, @@ -157,23 +157,31 @@ class OpenAIResponsesHandler(BaseTranslation): Handles both string input and list of message objects. """ input_data: Final[str | ResponseInputParam | None] = data.get("input") - tools_to_check: Final[list[ChatCompletionToolParam]] = [] if input_data is None: return data structured_messages: Final = self.get_structured_messages(data) + raw_tools: Final = data.get("tools") + original_tools: Final[tuple[Mapping[str, object], ...]] = ( + tuple(raw_tools) if isinstance(raw_tools, list) else () + ) + flattened_tool_groups: Final = tuple( + form.chat_tools for form in LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(original_tools) + ) + flattened_tools: Final = tuple( + cast(ChatCompletionToolParam, tool) # cast-ok: mcp tools ride along in the guardrail's tool list + for group in flattened_tool_groups + for tool in group + ) + tools_to_check: Final[list[ChatCompletionToolParam]] = list( # mutable-ok: guardrail inputs want a list + copy.deepcopy(flattened_tools) + ) # Handle simple string input if isinstance(input_data, str): inputs = GenericGuardrailAPIInputs(texts=[input_data]) - original_tools: list[dict[str, object]] = [] - - # Extract and transform tools if present - if "tools" in data and data["tools"]: - original_tools = list(data["tools"]) - self._extract_and_transform_tools(data["tools"], tools_to_check) - if tools_to_check: - inputs["tools"] = tools_to_check + if tools_to_check: + inputs["tools"] = tools_to_check if structured_messages: inputs["structured_messages"] = structured_messages # Include model information if available @@ -189,7 +197,9 @@ class OpenAIResponsesHandler(BaseTranslation): ) guardrailed_texts = guardrailed_inputs.get("texts", []) data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data - self._apply_guardrailed_tools_to_data(data, original_tools, guardrailed_inputs.get("tools")) + self._apply_guardrailed_tools_to_data( + data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools") + ) verbose_proxy_logger.debug("OpenAI Responses API: Processed string input") return data @@ -200,7 +210,6 @@ class OpenAIResponsesHandler(BaseTranslation): texts_to_check: Final[list[str]] = [] images_to_check: Final[list[str]] = [] task_mappings: Final[list[tuple[int, int | None]]] = [] - original_tools_list: Final[list[dict[str, object]]] = list(data.get("tools") or []) # Step 1: Extract all text content, images, and tools for msg_idx, message in enumerate(input_data): @@ -212,10 +221,6 @@ class OpenAIResponsesHandler(BaseTranslation): task_mappings=task_mappings, ) - # Extract and transform tools if present - if "tools" in data and data["tools"]: - self._extract_and_transform_tools(data["tools"], tools_to_check) - # Step 2: Apply guardrail to all texts in batch if texts_to_check: inputs = GenericGuardrailAPIInputs(texts=texts_to_check) @@ -238,9 +243,7 @@ class OpenAIResponsesHandler(BaseTranslation): guardrailed_texts = guardrailed_inputs.get("texts", []) self._apply_guardrailed_tools_to_data( - data, - original_tools_list, - guardrailed_inputs.get("tools"), + data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools") ) # Step 3: Map guardrail responses back to original input structure @@ -267,73 +270,18 @@ class OpenAIResponsesHandler(BaseTranslation): names.append(str(tool["server_label"])) return names - def _extract_and_transform_tools( - self, - tools: list[FunctionToolParam | OpenAIMcpServerTool], - tools_to_check: list[ChatCompletionToolParam], - ) -> None: - """ - Extract and transform tools from Responses API format to Chat Completion format. - - Uses the LiteLLM transformation function to convert Responses API tools - to Chat Completion tools that can be passed to guardrails. - """ - if tools is not None and isinstance(tools, list): - # Transform Responses API tools to Chat Completion tools - ( - transformed_tools, - _, - ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(tools) - tools_to_check.extend(cast(list[ChatCompletionToolParam], transformed_tools)) - - def _remap_tools_to_responses_api_format(self, guardrailed_tools: list[Any]) -> list[dict[str, object]]: - """ - Remap guardrail-returned tools (Chat Completion format) back to - Responses API request tool format. - """ - return LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools( - guardrailed_tools - ) - - def _merge_tools_after_guardrail( - self, - original_tools: list[dict[str, object]], - remapped: list[dict[str, object]], - ) -> list[dict[str, object]]: - """ - Merge remapped guardrailed tools with original tools that were not sent - to the guardrail (e.g. web_search, web_search_preview), preserving order. - Tools a guardrail appended (``remapped`` longer than ``original_tools``) - have no original slot and are kept so an injected tool is not dropped. - """ - if not original_tools: - return remapped - result: Final[list[dict[str, object]]] = [] - j = 0 - for tool in original_tools: - if isinstance(tool, dict) and tool.get("type") in ( - "web_search", - "web_search_preview", - ): - result.append(tool) - else: - if j < len(remapped): - result.append(remapped[j]) - j += 1 - # Keep guardrail-appended tools that matched no original slot above. - result.extend(remapped[j:]) - return result - def _apply_guardrailed_tools_to_data( self, data: dict, - original_tools: list[dict[str, object]], - guardrailed_tools: list[ChatCompletionToolParam] | None, + original_tools: Sequence[Mapping[str, object]], + flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]], + guardrailed_tools: Sequence[ChatCompletionToolParam] | None, ) -> None: - """Remap guardrailed tools to Responses API format and merge with original, then set data['tools'].""" - if guardrailed_tools is not None: - remapped: Final = self._remap_tools_to_responses_api_format(guardrailed_tools) - data["tools"] = self._merge_tools_after_guardrail(original_tools, remapped) + if guardrailed_tools is None: + return + data["tools"] = list( # mutable-ok: downstream wants a list # rebind-ok: in-place request rewrite + merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools) + ) def _extract_input_text_and_images( self, diff --git a/litellm/llms/openai/responses/guardrail_translation/tool_merge.py b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py new file mode 100644 index 00000000000..3ae951d3f61 --- /dev/null +++ b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py @@ -0,0 +1,177 @@ +from collections.abc import Iterable, Mapping, Sequence +from itertools import accumulate, chain +from types import MappingProxyType +from typing import Final, TypeAlias + +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_logger +from litellm.responses.litellm_completion_transformation.transformation import ( + NAMESPACE_DESCRIPTION_SEPARATOR, + LiteLLMCompletionResponsesConfig, +) + +Tool: TypeAlias = Mapping[str, object] +IndexedKey: TypeAlias = tuple[str, int] + +_TOOL_ADAPTER: Final = TypeAdapter(dict[str, object]) +_CHAT_TOOL_TOP_LEVEL_KEYS: Final = frozenset({"type", "function"}) + + +def _as_tool(value: object) -> Tool | None: + try: + return _TOOL_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _validated_tools(values: Iterable[object]) -> tuple[Tool, ...]: + validated: Final = tuple(map(_as_tool, values)) + dropped: Final = sum(tool is None for tool in validated) + if dropped: + verbose_logger.warning("Dropping %d guardrail-returned tools that are not objects", dropped) + return tuple(tool for tool in validated if tool is not None) + + +def _is_function(tool: Tool) -> bool: + return tool.get("type") == "function" + + +def _chat_tool_key(tool: Tool) -> str: + tool_type: Final = str(tool.get("type") or "") + function: Final = _as_tool(tool.get("function")) + if function is not None: + return f"{tool_type}:{function.get('name') or ''}" + return f"{tool_type}:{tool.get('server_label') or tool.get('name') or ''}" + + +def _indexed_keys(tools: Sequence[Tool]) -> tuple[IndexedKey, ...]: + keys: Final = tuple(_chat_tool_key(tool) for tool in tools) + return tuple((key, keys[:position].count(key)) for position, key in enumerate(keys)) + + +def _namespace_members(namespace: Tool) -> tuple[Tool, ...]: + members: Final = namespace.get("tools") + if not isinstance(members, Sequence) or isinstance(members, (str, bytes)): + return () + return tuple(member for member in map(_as_tool, members) if member is not None) + + +def _function_fields(tool: Tool) -> Tool: + function: Final = _as_tool(tool.get("function")) + return function if function is not None else MappingProxyType({}) + + +def _without_namespace_prefix(key: str, value: object, prefix: str) -> object: + if key != "description" or not isinstance(value, str) or not value.startswith(prefix): + return value + return value[len(prefix) :] + + +def _rebuilt_member(member: Tool, flattened: Tool, guardrailed: Tool, namespace_description: str) -> Tool: + flattened_function: Final = _function_fields(flattened) + prefix: Final = f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}" if namespace_description else "" + changed_function: Final = MappingProxyType( + { + key: _without_namespace_prefix(key, value, prefix) + for key, value in _function_fields(guardrailed).items() + if flattened_function.get(key) != value + } + ) + changed_extras: Final = MappingProxyType( + { + key: value + for key, value in guardrailed.items() + if key not in _CHAT_TOOL_TOP_LEVEL_KEYS and flattened.get(key) != value + } + ) + return {**member, **changed_extras, **changed_function} # mutable-ok: json.dumps rejects MappingProxyType + + +def _rebuilt_function_members( + function_members: Sequence[Tool], + flattened_group: Sequence[Tool], + group_keys: Sequence[IndexedKey], + guardrailed_by_key: Mapping[IndexedKey, Tool], + namespace_description: str, +) -> tuple[Tool | None, ...]: + return tuple( + None + if key not in guardrailed_by_key + else member + if guardrailed_by_key[key] == flattened + else _rebuilt_member(member, flattened, guardrailed_by_key[key], namespace_description) + for member, flattened, key in zip(function_members, flattened_group, group_keys) + ) + + +def _rebuilt_namespace( + original: Tool, + members: Sequence[Tool], + flattened_group: Sequence[Tool], + group_keys: Sequence[IndexedKey], + guardrailed_by_key: Mapping[IndexedKey, Tool], +) -> tuple[Tool, ...]: + namespace_description: Final = str(original.get("description") or "") + rebuilt_functions: Final = iter( + _rebuilt_function_members( + tuple(member for member in members if _is_function(member)), + flattened_group, + group_keys, + guardrailed_by_key, + namespace_description, + ) + ) + rebuilt_members: Final = tuple( + rebuilt + for rebuilt in (next(rebuilt_functions) if _is_function(member) else member for member in members) + if rebuilt is not None + ) + if not rebuilt_members: + return () + return ({**original, "tools": list(rebuilt_members)},) # mutable-ok: json.dumps needs a plain dict and list + + +def _merged_original( + original: Tool, + flattened_group: Sequence[Tool], + group_keys: Sequence[IndexedKey], + guardrailed_by_key: Mapping[IndexedKey, Tool], +) -> tuple[Tool, ...]: + if not group_keys: + return (original,) + guardrailed_group: Final = tuple(guardrailed_by_key[key] for key in group_keys if key in guardrailed_by_key) + if guardrailed_group == tuple(flattened_group): + return (original,) + if not guardrailed_group: + return () + members: Final = _namespace_members(original) if original.get("type") == "namespace" else () + if members and sum(map(_is_function, members)) == len(flattened_group): + return _rebuilt_namespace(original, members, flattened_group, group_keys, guardrailed_by_key) + return tuple( + LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools(guardrailed_group) + ) + + +def merge_guardrailed_tools( + original_tools: Sequence[Tool], + flattened_groups: Sequence[Sequence[Tool]], + guardrailed_tools: Iterable[object], +) -> tuple[Tool, ...]: + guardrailed: Final = _validated_tools(guardrailed_tools) + flattened_keys: Final = _indexed_keys(tuple(chain.from_iterable(flattened_groups))) + guardrailed_keys: Final = _indexed_keys(guardrailed) + guardrailed_by_key: Final = MappingProxyType(dict(zip(guardrailed_keys, guardrailed))) + group_ends: Final = tuple(accumulate(len(group) for group in flattened_groups)) + group_key_slices: Final = tuple( + flattened_keys[end - len(group) : end] for group, end in zip(flattened_groups, group_ends) + ) + merged_originals: Final = chain.from_iterable( + _merged_original(original, group, group_keys, guardrailed_by_key) + for original, group, group_keys in zip(original_tools, flattened_groups, group_key_slices) + ) + owned_keys: Final = frozenset(flattened_keys) + appended: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools( + tuple(tool for key, tool in zip(guardrailed_keys, guardrailed) if key not in owned_keys) + ) + return tuple(chain(merged_originals, appended)) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 5f3e88bb12f..9f91d7527cb 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -6,6 +6,7 @@ import json import re import uuid from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -102,6 +103,15 @@ from .custom_tools import ( NamespaceNameMap: TypeAlias = Mapping[str, tuple[str, str]] NamespaceTool: TypeAlias = Mapping[str, object] ResponseTools: TypeAlias = Sequence[Mapping[str, object]] | None +ChatToolParam: TypeAlias = ChatCompletionToolParam | OpenAIMcpServerTool +NAMESPACE_DESCRIPTION_SEPARATOR: Final = "\n\n" + + +@dataclass(frozen=True, slots=True) +class ResponsesToolChatForm: + chat_tools: tuple[ChatToolParam, ...] + web_search_options: OpenAIWebSearchOptions | None + if TYPE_CHECKING: from openai.types.responses.response_apply_patch_tool_call import ( @@ -1771,7 +1781,7 @@ class LiteLLMCompletionResponsesConfig: tool_name: Final = str(namespace_tool.get("name") or "") raw_description: Final = str(namespace_tool.get("description") or "") description: Final = ( - f"{namespace_description}\n\n{raw_description}" + f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}{raw_description}" if nested and namespace_description and raw_description else namespace_description if nested and namespace_description @@ -1837,9 +1847,82 @@ class LiteLLMCompletionResponsesConfig: + ", ".join(sorted(conflicting_tool_names)) ) + @staticmethod + def _responses_tool_to_chat_form(tool: Mapping[str, object]) -> ResponsesToolChatForm: + tool_type: Final = tool.get("type") + if tool_type == "mcp": + return ResponsesToolChatForm(chat_tools=(cast(OpenAIMcpServerTool, tool),), web_search_options=None) + if tool_type == "web_search_preview" or tool_type == "web_search": + _search_context_size: Final[Literal["low", "medium", "high"]] = cast( + Literal["low", "medium", "high"], tool.get("search_context_size") + ) + _user_location: Final[OpenAIWebSearchUserLocation | None] = cast( + OpenAIWebSearchUserLocation | None, + tool.get("user_location") or None, + ) + return ResponsesToolChatForm( + chat_tools=(), + web_search_options=OpenAIWebSearchOptions( + search_context_size=_search_context_size, + user_location=_user_location, + ), + ) + if tool_type == "function": + typed_tool: Final = cast(FunctionToolParam, tool) + raw_parameters: Final = typed_tool.get("parameters", {}) or {} + parameters: Final = ( + {**raw_parameters} # mutable-ok: json.dumps rejects MappingProxyType + if "type" in raw_parameters + else {**raw_parameters, "type": "object"} # mutable-ok: json.dumps rejects MappingProxyType + ) + chat_completion_tool: Final[dict[str, object]] = { + "type": "function", + "function": { + "name": typed_tool.get("name") or "", + "description": typed_tool.get("description") or "", + "parameters": parameters, + "strict": typed_tool.get("strict", False) or False, + }, + } + if tool.get("cache_control"): + chat_completion_tool["cache_control"] = tool.get("cache_control") + if tool.get("defer_loading"): + chat_completion_tool["defer_loading"] = tool.get("defer_loading") + if tool.get("allowed_callers"): + chat_completion_tool["allowed_callers"] = tool.get("allowed_callers") + if tool.get("input_examples"): + chat_completion_tool["input_examples"] = tool.get("input_examples") + return ResponsesToolChatForm( + chat_tools=(cast(ChatCompletionToolParam, chat_completion_tool),), web_search_options=None + ) + if tool_type == "namespace": + return ResponsesToolChatForm( + chat_tools=LiteLLMCompletionResponsesConfig._namespace_chat_tools(tool), web_search_options=None + ) + if tool_type == "custom": + converted: Final = convert_custom_tool_to_function_tool(tool) + return ResponsesToolChatForm(chat_tools=() if converted is None else (converted,), web_search_options=None) + if tool_type in ("computer_use", "image_generation", "shell"): + # Drop unsupported Responses-API-only tool types that have no + # Chat Completions equivalent. Passing them through verbatim + # causes providers to reject the request with "'function' is a + # required property". + verbose_logger.warning( + "Dropping Responses API tool of type '%s': it has no Chat Completions " + "equivalent and the target provider would reject the request.", + tool_type, + ) + return ResponsesToolChatForm(chat_tools=(), web_search_options=None) + return ResponsesToolChatForm(chat_tools=(cast(ChatToolParam, tool),), web_search_options=None) + + @staticmethod + def responses_tools_to_chat_forms(tools: ResponseTools) -> tuple[ResponsesToolChatForm, ...]: + LiteLLMCompletionResponsesConfig._validate_namespace_name_collisions(tools) + return tuple(LiteLLMCompletionResponsesConfig._responses_tool_to_chat_form(tool) for tool in tools or ()) + @staticmethod def transform_responses_api_tools_to_chat_completion_tools( - tools: list[FunctionToolParam | OpenAIMcpServerTool] | None, + tools: ResponseTools, ) -> tuple[ list[ChatCompletionToolParam | OpenAIMcpServerTool], OpenAIWebSearchOptions | None, @@ -1849,73 +1932,16 @@ class LiteLLMCompletionResponsesConfig: """ if tools is None: return [], None - LiteLLMCompletionResponsesConfig._validate_namespace_name_collisions(tools) - chat_completion_tools: Final[list[ChatCompletionToolParam | OpenAIMcpServerTool]] = [] - web_search_options: OpenAIWebSearchOptions | None = None - for tool in tools: - if tool.get("type") == "mcp": - chat_completion_tools.append(cast(OpenAIMcpServerTool, tool)) - elif tool.get("type") == "web_search_preview" or tool.get("type") == "web_search": - _search_context_size: Literal["low", "medium", "high"] = cast( - Literal["low", "medium", "high"], tool.get("search_context_size") - ) - _user_location: OpenAIWebSearchUserLocation | None = cast( - OpenAIWebSearchUserLocation | None, - tool.get("user_location") or None, - ) - web_search_options = OpenAIWebSearchOptions( - search_context_size=_search_context_size, - user_location=_user_location, - ) - elif tool.get("type") == "function": - typed_tool = cast(FunctionToolParam, tool) - # Ensure parameters has "type": "object" as required by providers like Anthropic - parameters = dict(typed_tool.get("parameters", {}) or {}) - if not parameters or "type" not in parameters: - parameters["type"] = "object" - chat_completion_tool: dict[str, object] = { - "type": "function", - "function": { - "name": typed_tool.get("name") or "", - "description": typed_tool.get("description") or "", - "parameters": parameters, - "strict": typed_tool.get("strict", False) or False, - }, - } - if tool.get("cache_control"): - chat_completion_tool["cache_control"] = tool.get("cache_control") - if tool.get("defer_loading"): - chat_completion_tool["defer_loading"] = tool.get("defer_loading") - if tool.get("allowed_callers"): - chat_completion_tool["allowed_callers"] = tool.get("allowed_callers") - if tool.get("input_examples"): - chat_completion_tool["input_examples"] = tool.get("input_examples") - chat_completion_tools.append(cast(ChatCompletionToolParam, chat_completion_tool)) - elif tool.get("type") == "namespace": - chat_completion_tools.extend(LiteLLMCompletionResponsesConfig._namespace_chat_tools(tool)) - elif tool.get("type") == "custom": - converted = convert_custom_tool_to_function_tool(tool) - if converted is not None: - chat_completion_tools.append(converted) - else: - _tool_type = tool.get("type") - if _tool_type in ("computer_use", "image_generation", "shell"): - # Drop unsupported Responses-API-only tool types that have no - # Chat Completions equivalent. Passing them through verbatim - # causes providers to reject the request with "'function' is a - # required property". - verbose_logger.warning( - "Dropping Responses API tool of type '%s': it has no Chat Completions " - "equivalent and the target provider would reject the request.", - _tool_type, - ) - continue - chat_completion_tools.append(cast(ChatCompletionToolParam | OpenAIMcpServerTool, tool)) - return chat_completion_tools, web_search_options + forms: Final = LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(tools) + web_search_options: Final = next( + (form.web_search_options for form in reversed(forms) if form.web_search_options is not None), + None, + ) + return [chat_tool for form in forms for chat_tool in form.chat_tools], web_search_options @staticmethod def transform_chat_completion_tool_params_to_responses_api_tools( - chat_completion_tools: list[ChatCompletionToolParam | OpenAIMcpServerTool] | None, + chat_completion_tools: Sequence[Mapping[str, object]] | None, ) -> list[dict[str, object]]: """ Transform Chat Completion tool params (e.g. from guardrail output) back to @@ -1926,9 +1952,6 @@ class LiteLLMCompletionResponsesConfig: return [] result: Final[list[dict[str, object]]] = [] for tool in chat_completion_tools: - if not isinstance(tool, dict): - result.append(tool) - continue if tool.get("type") == "function": fn = cast(_ToolFunctionDefinition, tool.get("function") or {}) parameters = dict(fn.get("parameters", {}) or {}) diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 9b1cc977a64..935fed18a79 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -177,7 +177,7 @@ "limit": 8 }, "RUF019": { - "limit": 31 + "limit": 29 }, "RUF046": { "limit": 4 diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index 315b6948bd8..d071ef78c2d 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -5,6 +5,8 @@ Tests the handler's ability to process input/output for the Responses API with guardrail transformations. """ +import copy +from collections.abc import Callable from typing import Any, List, Literal, Optional, Tuple from unittest.mock import AsyncMock, MagicMock @@ -19,6 +21,10 @@ from litellm.llms import get_guardrail_translation_mapping from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, ) +from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools +from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, +) from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.responses.main import GenericResponseOutputItem, OutputText from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs @@ -1287,14 +1293,14 @@ class TestOpenAIResponsesHandlerToolInjection: """A tool a guardrail injects must survive the write-back to Responses format.""" def test_merge_keeps_guardrail_appended_tool(self): - """_merge_tools_after_guardrail must not drop the extra appended tool.""" - handler = OpenAIResponsesHandler() + """merge_guardrailed_tools must not drop the extra appended tool.""" original = [{"type": "function", "name": "a"}] - remapped = [ - {"type": "function", "name": "a"}, - {"type": "function", "name": "b"}, + groups = [form.chat_tools for form in LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(original)] + guardrailed = [ + *groups[0], + {"type": "function", "function": {"name": "b", "description": "", "parameters": {"type": "object"}}}, ] - merged = handler._merge_tools_after_guardrail(original, remapped) + merged = merge_guardrailed_tools(original, groups, guardrailed) assert [t["name"] for t in merged] == ["a", "b"] @pytest.mark.asyncio @@ -1323,6 +1329,194 @@ class TestOpenAIResponsesHandlerToolInjection: assert "injected_tool" in names +class ToolEditingGuardrail(CustomGuardrail): + """Guardrail that rewrites the flattened chat tools it was handed through ``edit``""" + + def __init__(self, edit: Callable[[list[dict]], list[dict]], **kwargs): + super().__init__(**kwargs) + self.edit = edit + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Any | None = None, + ) -> GenericGuardrailAPIInputs: + inputs["tools"] = self.edit(list(inputs.get("tools") or [])) + return inputs + + +def _codex_request(input_value): + """A Responses API request shaped like what the Codex CLI sends when an MCP server is configured""" + return { + "model": "gpt-5.3-codex", + "input": input_value, + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Weather lookup", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + "strict": False, + }, + { + "type": "namespace", + "name": "mcp__confluence", + "description": "Confluence tools", + "tools": [ + { + "type": "function", + "name": "confluence_get_page", + "description": "Get a page", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + "strict": False, + }, + { + "type": "function", + "name": "confluence_search", + "description": "Search pages", + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, + "strict": False, + }, + ], + }, + { + "type": "custom", + "name": "apply_patch", + "description": "Apply a patch", + "format": {"type": "grammar", "syntax": "lark", "definition": 'start: "x"'}, + }, + {"type": "web_search"}, + ], + } + + +def _tool_named(tools, name): + return next(tool for tool in tools if tool.get("name") == name) + + +class TestOpenAIResponsesHandlerNamespaceTools: + """Codex sends MCP tools as ``namespace`` tools; a guardrail must never flatten them (GH #39183)""" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "input_value", + ["hi", [{"role": "user", "content": "hi", "type": "message"}]], + ids=["string_input", "list_input"], + ) + async def test_pass_through_guardrail_leaves_tools_untouched(self, input_value): + data = _codex_request(input_value) + expected_tools = copy.deepcopy(data["tools"]) + + result = await OpenAIResponsesHandler().process_input_messages( + data, MockPassThroughGuardrail(guardrail_name="test") + ) + + assert result["tools"] == expected_tools + + @pytest.mark.asyncio + async def test_appending_guardrail_keeps_namespace_and_adds_tool(self): + data = _codex_request("hi") + expected_tools = copy.deepcopy(data["tools"]) + + result = await OpenAIResponsesHandler().process_input_messages( + data, ToolAppendingGuardrail(guardrail_name="test") + ) + + assert result["tools"][:-1] == expected_tools + assert result["tools"][-1]["type"] == "function" + assert result["tools"][-1]["name"] == "injected_tool" + + @pytest.mark.asyncio + async def test_dropping_one_member_prunes_only_that_member(self): + data = _codex_request("hi") + expected_tools = copy.deepcopy(data["tools"]) + guardrail = ToolEditingGuardrail( + edit=lambda tools: [t for t in tools if t["function"]["name"] != "mcp__confluence__confluence_search"], + guardrail_name="test", + ) + + result = await OpenAIResponsesHandler().process_input_messages(data, guardrail) + + namespace = _tool_named(result["tools"], "mcp__confluence") + assert [member["name"] for member in namespace["tools"]] == ["confluence_get_page"] + assert namespace["tools"][0] == expected_tools[1]["tools"][0] + assert [t for t in result["tools"] if t is not namespace] == [expected_tools[0], *expected_tools[2:]] + + @pytest.mark.asyncio + async def test_editing_a_member_lands_on_that_member_without_the_namespace_prefix(self): + data = _codex_request("hi") + expected_tools = copy.deepcopy(data["tools"]) + + def redact_search(tools): + for tool in tools: + if tool["function"]["name"] == "mcp__confluence__confluence_search": + tool["function"]["description"] = "Confluence tools\n\nREDACTED" + return tools + + result = await OpenAIResponsesHandler().process_input_messages( + data, ToolEditingGuardrail(edit=redact_search, guardrail_name="test") + ) + + namespace = _tool_named(result["tools"], "mcp__confluence") + assert namespace["tools"][0] == expected_tools[1]["tools"][0] + assert namespace["tools"][1] == {**expected_tools[1]["tools"][1], "description": "REDACTED"} + assert {k: v for k, v in namespace.items() if k != "tools"} == { + k: v for k, v in expected_tools[1].items() if k != "tools" + } + + @pytest.mark.asyncio + async def test_dropping_every_member_drops_the_namespace(self): + data = _codex_request("hi") + expected_tools = copy.deepcopy(data["tools"]) + guardrail = ToolEditingGuardrail( + edit=lambda tools: [t for t in tools if not t["function"]["name"].startswith("mcp__confluence__")], + guardrail_name="test", + ) + + result = await OpenAIResponsesHandler().process_input_messages(data, guardrail) + + assert result["tools"] == [expected_tools[0], *expected_tools[2:]] + + @pytest.mark.asyncio + async def test_edited_top_level_function_is_rewritten_in_place(self): + data = _codex_request("hi") + expected_tools = copy.deepcopy(data["tools"]) + + def rename_weather(tools): + for tool in tools: + if tool["function"]["name"] == "get_weather": + tool["function"]["description"] = "Weather lookup (guarded)" + return tools + + result = await OpenAIResponsesHandler().process_input_messages( + data, ToolEditingGuardrail(edit=rename_weather, guardrail_name="test") + ) + + assert result["tools"][0] == {**expected_tools[0], "description": "Weather lookup (guarded)"} + assert result["tools"][1:] == expected_tools[1:] + + +class TestOpenAIResponsesHandlerMalformedTools: + @pytest.mark.asyncio + async def test_request_tools_that_are_not_a_list_never_reach_the_guardrail(self): + handler = OpenAIResponsesHandler() + seen: list[list[dict]] = [] + + def record(tools): + seen.append(tools) + return tools + + guardrail = ToolEditingGuardrail(edit=record, guardrail_name="test") + data = {"input": "hi", "tools": {"type": "function", "name": "get_weather"}} + + result = await handler.process_input_messages(data, guardrail) + + assert seen == [[]] + assert result["input"] == "hi" + + class TestBuildBlockSseChunks: """build_block_sse_chunks turns a streaming ModifyResponseException into 200 SSE events""" diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py new file mode 100644 index 00000000000..b80dd0b36aa --- /dev/null +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py @@ -0,0 +1,144 @@ +""" +Unit tests for merge_guardrailed_tools, which writes guardrail-returned chat tools back onto the +Responses API request tools they were flattened from +""" + +import copy + +from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools +from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, +) + + +def _groups(tools): + return [form.chat_tools for form in LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(tools)] + + +def _flat(groups): + return [chat_tool for group in groups for chat_tool in group] + + +def _function(name, description=""): + return {"type": "function", "name": name, "description": description, "parameters": {"type": "object"}} + + +def test_unchanged_tools_come_back_as_the_original_objects(): + original = [ + _function("a"), + {"type": "namespace", "name": "ns", "description": "NS", "tools": [_function("x"), _function("y")]}, + {"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp"}, + {"type": "web_search"}, + ] + groups = _groups(original) + + merged = merge_guardrailed_tools(original, groups, _flat(groups)) + + assert list(merged) == original + assert all(merged_tool is original_tool for merged_tool, original_tool in zip(merged, original)) + + +def test_guardrail_reordering_unchanged_tools_keeps_request_order(): + original = [_function("a"), {"type": "namespace", "name": "ns", "tools": [_function("x")]}, {"type": "web_search"}] + groups = _groups(original) + + merged = merge_guardrailed_tools(original, groups, list(reversed(_flat(groups)))) + + assert list(merged) == original + + +def test_duplicate_function_names_are_matched_by_ordinal(): + original = [_function("dup", "first"), _function("dup", "second")] + groups = _groups(original) + + merged = merge_guardrailed_tools(original, groups, _flat(groups)[:1]) + + assert list(merged) == [original[0]] + + +def test_edited_mcp_tool_is_rewritten(): + original = [{"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp"}] + groups = _groups(original) + edited = [{**groups[0][0], "allowed_tools": ["read_wiki_structure"]}] + + merged = merge_guardrailed_tools(original, groups, edited) + + assert list(merged) == edited + + +def test_injected_tool_lands_after_the_request_tools_when_request_had_none(): + injected = {"type": "function", "function": {"name": "b", "description": "d", "parameters": {"type": "object"}}} + + merged = merge_guardrailed_tools([], [], [injected]) + + assert list(merged) == [ + {"type": "function", "name": "b", "description": "d", "parameters": {"type": "object"}, "strict": False} + ] + + +def test_empty_guardrail_output_keeps_only_tools_never_sent_to_the_guardrail(): + original = [_function("a"), {"type": "web_search"}, {"type": "namespace", "name": "ns", "tools": [_function("x")]}] + + merged = merge_guardrailed_tools(original, _groups(original), []) + + assert list(merged) == [{"type": "web_search"}] + + +def test_member_edit_strips_only_the_namespace_description_prefix(): + original = [{"type": "namespace", "name": "ns", "description": "NS", "tools": [_function("x", "X doc")]}] + groups = _groups(original) + assert groups[0][0]["function"]["description"] == "NS\n\nX doc" + edited = [{**groups[0][0], "function": {**groups[0][0]["function"], "description": "NS\n\nX doc (guarded)"}}] + + merged = merge_guardrailed_tools(original, groups, edited) + + assert list(merged) == [ + {"type": "namespace", "name": "ns", "description": "NS", "tools": [_function("x", "X doc (guarded)")]} + ] + + +def test_namespace_keeps_a_non_function_member_when_a_function_member_is_edited(): + custom_member = {"type": "custom", "name": "grep", "description": "Grep", "format": {"type": "text"}} + original = [ + {"type": "namespace", "name": "ns", "description": "NS", "tools": [_function("read", "Read"), custom_member]} + ] + groups = _groups(original) + edited = copy.deepcopy(_flat(groups)) + edited[0]["function"]["description"] = "NS\n\nEDITED" + + merged = merge_guardrailed_tools(original, groups, edited) + + assert len(merged) == 1 + assert [member["name"] for member in merged[0]["tools"]] == ["read", "grep"] + assert merged[0]["tools"][0]["description"] == "EDITED" + assert merged[0]["tools"][1] == custom_member + + +def test_member_extras_edited_by_the_guardrail_land_on_that_member(): + original = [{"type": "namespace", "name": "ns", "tools": [_function("read")]}] + groups = _groups(original) + edited = copy.deepcopy(_flat(groups)) + edited[0]["cache_control"] = {"type": "ephemeral"} + + merged = merge_guardrailed_tools(original, groups, edited) + + assert merged[0]["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert merged[0]["tools"][0]["name"] == "read" + + +def test_guardrail_output_is_read_once(): + original = [_function("a"), {"type": "namespace", "name": "ns", "tools": [_function("x")]}] + groups = _groups(original) + + merged = merge_guardrailed_tools(original, groups, (chat_tool for chat_tool in _flat(groups))) + + assert list(merged) == original + + +def test_non_object_guardrail_items_are_dropped(): + original = [_function("a")] + groups = _groups(original) + + merged = merge_guardrailed_tools(original, groups, [*_flat(groups), "junk", None]) + + assert list(merged) == original diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index b2b8eb5da80..2068f10ea2d 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1928,6 +1928,19 @@ class TestToolTransformation: assert result_tool["function"]["parameters"]["type"] == "object" assert "properties" in result_tool["function"]["parameters"] + def test_transform_function_tools_parameters_keep_client_key_order(self): + tools = [ + {"type": "function", "name": "a", "parameters": {"properties": {"arg": {"type": "string"}}, "required": ["arg"]}}, + {"type": "function", "name": "b", "parameters": {"type": "object", "properties": {}}}, + ] + + result_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) + + assert list(result_tools[0]["function"]["parameters"]) == ["properties", "required", "type"] + assert list(result_tools[1]["function"]["parameters"]) == ["type", "properties"] + def test_transform_function_tools_empty_parameters(self): """Test that empty parameters get 'type': 'object' added""" function_tool = { diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 52cb9628252..970a44cd4fa 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 22364 + "limit": 22340 }, "LIT002": { - "limit": 26777 + "limit": 26770 }, "LIT003": { "limit": 269 @@ -15,7 +15,7 @@ "limit": 0 }, "LIT006": { - "limit": 1039 + "limit": 1038 }, "LIT007": { "limit": 0 @@ -27,10 +27,10 @@ "limit": 0 }, "LIT010": { - "limit": 16507 + "limit": 16503 }, "LIT011": { - "limit": 5535 + "limit": 5534 }, "LIT012": { "limit": 4495 From 49c69c46b25af2dd962b322bd6c8e6c5668546c6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:51:27 -0700 Subject: [PATCH 087/175] fix(bedrock): drop the OpenAI base suffix from BEDROCK_MANTLE_API_BASE before the mantle messages path --- litellm/llms/bedrock/common_utils.py | 15 +++++++++++++-- tests/test_litellm/llms/bedrock/test_mantle.py | 12 +++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 048d023a1bd..1e5329c90dd 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -748,6 +748,15 @@ def strip_bedrock_throughput_suffix(model: str) -> str: MANTLE_MESSAGES_PATH: Final = "/anthropic/v1/messages" +_MANTLE_OPENAI_BASE_SUFFIXES: Final = ("/openai/v1", "/v1") + + +def _mantle_api_base_from_env() -> str | None: + env_base: Final = get_secret_str("BEDROCK_MANTLE_API_BASE") + if env_base is None: + return None + base: Final = env_base.rstrip("/") + return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base) def build_mantle_messages_url( @@ -762,9 +771,11 @@ def build_mantle_messages_url( private VPC / VPCE / GovCloud Mantle endpoints are reachable; otherwise falls back to the public regional host. The mantle messages path is appended unless the override already carries it, - so callers can pass either the host or the full messages URL. + so callers can pass either the host or the full messages URL. The env var is + shared with the OpenAI-surface ``bedrock_mantle/*`` routes, which need it to + carry their ``/v1`` or ``/openai/v1`` base, so that suffix is dropped first. """ - override: Final = api_base or aws_bedrock_runtime_endpoint or get_secret_str("BEDROCK_MANTLE_API_BASE") + override: Final = api_base or aws_bedrock_runtime_endpoint or _mantle_api_base_from_env() if override: base: Final = override.rstrip("/") if base.endswith(MANTLE_MESSAGES_PATH): diff --git a/tests/test_litellm/llms/bedrock/test_mantle.py b/tests/test_litellm/llms/bedrock/test_mantle.py index d1d1ba447fb..09be2118001 100644 --- a/tests/test_litellm/llms/bedrock/test_mantle.py +++ b/tests/test_litellm/llms/bedrock/test_mantle.py @@ -203,8 +203,12 @@ _ENV_ENDPOINT = "https://bedrock-mantle.us-east-1.api.aws.internal.example.com" @pytest.mark.parametrize("config_cls", [AmazonMantleConfig, AmazonMantleMessagesConfig]) -def test_mantle_url_honors_bedrock_mantle_api_base_env(monkeypatch, config_cls): - monkeypatch.setenv("BEDROCK_MANTLE_API_BASE", _ENV_ENDPOINT) +@pytest.mark.parametrize( + "env_value", + [_ENV_ENDPOINT, f"{_ENV_ENDPOINT}/", f"{_ENV_ENDPOINT}/v1", f"{_ENV_ENDPOINT}/openai/v1"], +) +def test_mantle_url_honors_bedrock_mantle_api_base_env(monkeypatch, config_cls, env_value): + monkeypatch.setenv("BEDROCK_MANTLE_API_BASE", env_value) url = config_cls().get_complete_url( api_base=None, api_key=None, @@ -223,7 +227,9 @@ def test_mantle_url_honors_bedrock_mantle_api_base_env(monkeypatch, config_cls): (None, {"aws_region_name": "us-gov-west-1", "aws_bedrock_runtime_endpoint": _VPC_ENDPOINT}), ], ) -def test_mantle_url_explicit_endpoint_beats_bedrock_mantle_api_base_env(monkeypatch, config_cls, api_base, optional_params): +def test_mantle_url_explicit_endpoint_beats_bedrock_mantle_api_base_env( + monkeypatch, config_cls, api_base, optional_params +): monkeypatch.setenv("BEDROCK_MANTLE_API_BASE", _ENV_ENDPOINT) url = config_cls().get_complete_url( api_base=api_base, From cc2cbb36f326a87cd72421a4448349734f872201 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:57:45 -0700 Subject: [PATCH 088/175] fix(otel): stamp Langfuse root observation input and output from the request task --- litellm/integrations/otel/langfuse_logger.py | 69 ++++ litellm/integrations/otel/logger.py | 32 +- litellm/integrations/otel/mappers/langfuse.py | 8 +- litellm/integrations/otel/model/request_io.py | 90 +++++ litellm/litellm_core_utils/litellm_logging.py | 14 +- .../integrations/otel/test_langfuse_logger.py | 317 ++++++++++++++++++ 6 files changed, 519 insertions(+), 11 deletions(-) create mode 100644 litellm/integrations/otel/langfuse_logger.py create mode 100644 litellm/integrations/otel/model/request_io.py create mode 100644 tests/test_litellm/integrations/otel/test_langfuse_logger.py diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py new file mode 100644 index 00000000000..9986eae4d0a --- /dev/null +++ b/litellm/integrations/otel/langfuse_logger.py @@ -0,0 +1,69 @@ +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_logger +from litellm.integrations.otel.logger import OpenTelemetryV2 +from litellm.integrations.otel.mappers.langfuse import LANGFUSE_OBSERVATION_INPUT, LANGFUSE_OBSERVATION_OUTPUT +from litellm.integrations.otel.model.request_io import request_input, response_output, stream_output +from litellm.integrations.otel.plumbing.context import request_root_span + +if TYPE_CHECKING: + from litellm.caching.dual_cache import DualCache + from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.utils import CallTypesLiteral, ModelResponseStream + +ROOT_OBSERVATION_IO_CALL_TYPES: Final = frozenset( + {"completion", "acompletion", "responses", "aresponses", "anthropic_messages", "aanthropic_messages"} +) + + +class LangfuseOpenTelemetryV2(OpenTelemetryV2): + """Stamps the request's input and output on the root observation while it is still recording. + + Langfuse shows a trace's input and output from its root observation. The proxy's root span ends + when the response is sent, before the success callback runs, so the stamps have to come from the + request-task hooks: input at pre-call, output at post-call success or at the end of the stream. + """ + + async def async_pre_call_hook( + self, + user_api_key_dict: "UserAPIKeyAuth", + cache: "DualCache", + data: Mapping[str, object], + call_type: "CallTypesLiteral", + ) -> None: + await super().async_pre_call_hook(user_api_key_dict, cache, data, call_type) + if call_type in ROOT_OBSERVATION_IO_CALL_TYPES: + self._stamp_root(LANGFUSE_OBSERVATION_INPUT, lambda: request_input(data)) + + async def async_post_call_success_hook( + self, + data: Mapping[str, object], + user_api_key_dict: "UserAPIKeyAuth", + response: object, + ) -> None: + self._stamp_root(LANGFUSE_OBSERVATION_OUTPUT, lambda: response_output(response)) + + async def async_post_call_streaming_iterator_hook( + self, + user_api_key_dict: "UserAPIKeyAuth", + response: "AsyncIterator[ModelResponseStream]", + request_data: Mapping[str, object], + ) -> "AsyncGenerator[ModelResponseStream, None]": + relayed: Final[list[ModelResponseStream]] = [] # mutable-ok: relayed as they arrive, assembled at end of stream + async for chunk in response: + relayed.append(chunk) + yield chunk + self._stamp_root(LANGFUSE_OBSERVATION_OUTPUT, lambda: stream_output(tuple(relayed), request_data)) + + def _stamp_root(self, key: str, render: Callable[[], str | None]) -> None: + root: Final = request_root_span() + if root is None or not root.is_recording(): + return + try: + value: Final = render() + except Exception: # noqa: BLE001 # telemetry must never fail the request it describes + verbose_logger.debug("otel v2 langfuse: could not render %s for the root observation", key, exc_info=True) + return + if value is not None: + root.set_attribute(key, value) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index d2a32ef73b6..4ab1c738488 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -4,6 +4,7 @@ from collections import OrderedDict from collections.abc import Callable, Iterator, Mapping, Sequence from contextlib import contextmanager from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast from opentelemetry.context import Context, attach, get_current @@ -722,14 +723,13 @@ class OpenTelemetryV2(CustomLogger): self, user_api_key_dict: "UserAPIKeyAuth", cache: "DualCache", - data: dict, + data: Mapping[str, object], call_type: "CallTypesLiteral", - ) -> dict: + ) -> None: self.seed_request_identity( user_api_key_dict, model=model_from_request_data(data), ) - return data def record_error_attributes_on_span( self, @@ -909,3 +909,29 @@ def phase_span(name: str) -> "Iterator[Span | None]": return with logger.start_phase_span(name) as span: yield span + + +def build_otel_v2_logger( + config: OpenTelemetryV2Config, + callback_name: str | None = None, + tracer_provider: TracerProvider | None = None, + logger_provider: LoggerProvider | None = None, + meter_provider: "MeterProvider | None" = None, + settings: Mapping[str, object] = MappingProxyType({}), +) -> OpenTelemetryV2: + return _logger_class(config)( + config=config, + callback_name=callback_name, + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + **settings, + ) + + +def _logger_class(config: OpenTelemetryV2Config) -> type[OpenTelemetryV2]: + if "langfuse" not in config.mapper_names or not config.capture_span_content: + return OpenTelemetryV2 + from litellm.integrations.otel.langfuse_logger import LangfuseOpenTelemetryV2 + + return LangfuseOpenTelemetryV2 diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index 6d4f1b4fd0a..01063d85355 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -11,6 +11,7 @@ the JSON-serialized payloads. ``_llm_call`` just applies both tables. import json from collections.abc import Callable +from typing import Final from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData from litellm.integrations.otel.mappers.utils import ( @@ -25,6 +26,9 @@ from litellm.integrations.otel.model.payloads import ( LLMUsage, ) +LANGFUSE_OBSERVATION_INPUT: Final = "langfuse.observation.input" +LANGFUSE_OBSERVATION_OUTPUT: Final = "langfuse.observation.output" + class LangfuseMapper: _LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { @@ -56,8 +60,8 @@ class LangfuseMapper: "langfuse.observation.model.parameters": lambda d: json_if( collect(LangfuseMapper._MODEL_PARAMS, d.request_params) ), - "langfuse.observation.input": lambda d: serialize_messages(d.messages_in), - "langfuse.observation.output": lambda d: serialize_messages(output_messages(d)), + LANGFUSE_OBSERVATION_INPUT: lambda d: serialize_messages(d.messages_in), + LANGFUSE_OBSERVATION_OUTPUT: lambda d: serialize_messages(output_messages(d)), "langfuse.observation.usage_details": lambda d: json_if(collect(LangfuseMapper._USAGE_FIELDS, d.usage)), "langfuse.observation.cost_details": lambda d: ( json.dumps({"total": d.response_cost}) if d.response_cost is not None else None diff --git a/litellm/integrations/otel/model/request_io.py b/litellm/integrations/otel/model/request_io.py new file mode 100644 index 00000000000..4e80fb91993 --- /dev/null +++ b/litellm/integrations/otel/model/request_io.py @@ -0,0 +1,90 @@ +from collections.abc import Mapping, Sequence +from typing import Final, Literal + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError +from typing_extensions import ReadOnly, TypedDict + +import litellm +from litellm.integrations.otel.mappers.utils import json_or_none +from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_stream, is_raw_sse_stream +from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse +from litellm.types.utils import ModelResponse, ModelResponseStream + +_SYSTEM_KEYS: Final = ("system", "instructions") +_TURNS: Final = TypeAdapter(tuple[object, ...]) +_MESSAGES: Final = TypeAdapter(list[object] | None) + + +class _Turn(TypedDict): + role: ReadOnly[str] + content: ReadOnly[object] + + +class _AnthropicMessage(BaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["message"] = Field(exclude=True) + role: str = "assistant" + content: object = None + + +def request_input(data: Mapping[str, object]) -> str | None: + turns: Final = data.get("messages", data.get("input")) + if turns is None: + return None + return json_or_none((*_system_turns(data), *_user_turns(turns))) + + +def _system_turns(data: Mapping[str, object]) -> tuple[_Turn, ...]: + return tuple(_Turn(role="system", content=data[key]) for key in _SYSTEM_KEYS if data.get(key) is not None) + + +def _user_turns(turns: object) -> tuple[object, ...]: + if isinstance(turns, str): + return (_Turn(role="user", content=turns),) + try: + return _TURNS.validate_python(turns) + except ValidationError: + return (_Turn(role="user", content=turns),) + + +def response_output(response: object) -> str | None: + match response: + case ModelResponse(): + return json_or_none(tuple(choice.message.model_dump(exclude_none=True) for choice in response.choices)) + case ResponsesAPIResponse(): + return json_or_none(response.model_dump(exclude_none=True).get("output")) + case _: + return _anthropic_message_output(response) + + +def _anthropic_message_output(message: object) -> str | None: + try: + parsed: Final = _AnthropicMessage.model_validate(message) + except ValidationError: + return None + return json_or_none((parsed.model_dump(),)) + + +def stream_output(chunks: Sequence[object], data: Mapping[str, object]) -> str | None: + if not chunks: + return None + if is_raw_sse_stream(chunks): + return response_output(assemble_anthropic_sse_stream(chunks)) + if all(isinstance(chunk, ModelResponseStream) for chunk in chunks): + return response_output(_assembled_chat_stream(chunks, data)) + return response_output(_completed_response(chunks)) + + +def _assembled_chat_stream(chunks: Sequence[object], data: Mapping[str, object]) -> object: + try: + return litellm.stream_chunk_builder( # pyright: ignore[reportUnknownMemberType] # upstream types chunks as a bare list + chunks=list(chunks), # mutable-ok: stream_chunk_builder takes a list + messages=_MESSAGES.validate_python(data.get("messages")), + ) + except (litellm.APIError, ValidationError): + return None + + +def _completed_response(chunks: Sequence[object]) -> ResponsesAPIResponse | None: + return next((chunk.response for chunk in reversed(chunks) if isinstance(chunk, ResponseCompletedEvent)), None) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9a6fb11f978..8e8575eff9a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4390,13 +4390,15 @@ def _init_custom_logger_compatible_class( from litellm.integrations.otel.model.config import is_otel_v2_enabled if is_otel_v2_enabled(): - from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger + from litellm.integrations.otel.model.config import OpenTelemetryV2Config for callback in _in_memory_loggers: - if type(callback) is OpenTelemetryV2: + if isinstance(callback, OpenTelemetryV2): return callback - otel_logger_v2: Final = OpenTelemetryV2( - **_get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) + otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) + otel_logger_v2: Final = build_otel_v2_logger( + config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings ) _in_memory_loggers.append(otel_logger_v2) _maybe_auto_initialize_arize_phoenix(_in_memory_loggers) @@ -4759,7 +4761,7 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom if not is_otel_v2_enabled(): return None - from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger from litellm.integrations.otel.presets import PRESET_BY_CALLBACK preset_fn: Final = PRESET_BY_CALLBACK.get(callback_name) @@ -4774,7 +4776,7 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None - v2_logger: Final = OpenTelemetryV2(config=config, callback_name=callback_name) + v2_logger: Final = build_otel_v2_logger(config=config, callback_name=callback_name) _in_memory_loggers.append(v2_logger) return v2_logger diff --git a/tests/test_litellm/integrations/otel/test_langfuse_logger.py b/tests/test_litellm/integrations/otel/test_langfuse_logger.py new file mode 100644 index 00000000000..af0597517dc --- /dev/null +++ b/tests/test_litellm/integrations/otel/test_langfuse_logger.py @@ -0,0 +1,317 @@ +"""Tests for ``LangfuseOpenTelemetryV2``: the root observation's input and output are stamped from the +request-task hooks, while the root span is still recording, so Langfuse can show them on the trace.""" + +import asyncio +import json +from collections.abc import AsyncIterator, Sequence +from typing import Final + +import pytest + +pytest.importorskip("opentelemetry") + +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 + +from litellm.caching.dual_cache import DualCache # noqa: E402 +from litellm.integrations.otel.langfuse_logger import LangfuseOpenTelemetryV2 # noqa: E402 +from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger # noqa: E402 +from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402 +from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402 +from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402 +from litellm.integrations.otel.plumbing import providers # noqa: E402 +from litellm.integrations.otel.plumbing.context import set_request_root_span # noqa: E402 +from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 # noqa: E402 +from litellm.proxy._types import UserAPIKeyAuth # noqa: E402 +from litellm.types.llms.openai import ( # noqa: E402 + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) +from litellm.types.utils import ( # noqa: E402 + Choices, + Delta, + Message, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) + +INPUT_ATTR: Final = "langfuse.observation.input" +OUTPUT_ATTR: Final = "langfuse.observation.output" +CHAT_DATA: Final = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "ping"}]} + + +@pytest.fixture(autouse=True) +def _reset_request_root_span(): + otel_context._request_root_span.set(None) + yield + otel_context._request_root_span.set(None) + + +def _logger(*, capture: str = "span_only", mappers: Sequence[str] = ("genai", "langfuse")): + cfg = OpenTelemetryV2Config(exporter="in_memory", mapper_names=list(mappers), capture_message_content=capture) + exporter = InMemorySpanExporter() + tracer_provider = providers.build_tracer_provider(cfg, exporter=exporter) + return build_otel_v2_logger(config=cfg, tracer_provider=tracer_provider), exporter + + +def _start_root(logger): + root = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + set_request_root_span(root) + return root + + +def _root_attrs(exporter): + by_name = {span.name: span for span in exporter.get_finished_spans()} + return dict(by_name[LITELLM_PROXY_REQUEST_SPAN_NAME].attributes or {}) + + +def _run_request(logger, data: dict, call_type: str, response: object): + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), data, call_type)) + asyncio.run(logger.async_post_call_success_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), response=response)) + root.end() + + +async def _relay(logger, chunks: Sequence[object], data: dict) -> list[object]: + async def source() -> AsyncIterator[object]: + for chunk in chunks: + yield chunk + + return [chunk async for chunk in logger.async_post_call_streaming_iterator_hook(UserAPIKeyAuth(), source(), data)] + + +def _run_stream(logger, data: dict, chunks: Sequence[object]) -> list[object]: + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), data, "acompletion")) + relayed = asyncio.run(_relay(logger, chunks, data)) + root.end() + return relayed + + +def _chat_chunk(content: str | None, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-1", + created=1, + model="gpt-5.4-mini", + choices=[StreamingChoices(index=0, delta=Delta(content=content), finish_reason=finish_reason)], + ) + + +def _responses_api_response() -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_1", + created_at=1, + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "pong", "annotations": []}], + } + ], + ) + + +def _anthropic_sse_frames() -> tuple[bytes, ...]: + events = ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "po"}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ng"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 2}}, + {"type": "message_stop"}, + ) + return tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events) + + +def test_chat_request_stamps_root_observation_input_and_output(): + logger, exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + + _run_request(logger, CHAT_DATA, "acompletion", response) + + attrs = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [{"role": "user", "content": "ping"}] + output = json.loads(attrs[OUTPUT_ATTR]) + assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] + + +def test_responses_request_folds_instructions_into_input_and_stamps_output_items(): + logger, exporter = _logger() + data = {"model": "gpt-5.4-mini", "instructions": "be terse", "input": "ping"} + + _run_request(logger, data, "aresponses", _responses_api_response()) + + attrs = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "ping"}, + ] + output = json.loads(attrs[OUTPUT_ATTR]) + assert output[0]["role"] == "assistant" + assert output[0]["content"][0]["text"] == "pong" + + +def test_anthropic_messages_request_folds_system_into_input_and_stamps_content_blocks(): + logger, exporter = _logger() + data = {"model": "claude-sonnet-4-5", "system": "be terse", "messages": [{"role": "user", "content": "ping"}]} + response = {"type": "message", "role": "assistant", "content": [{"type": "text", "text": "pong"}]} + + _run_request(logger, data, "aanthropic_messages", response) + + attrs = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "ping"}, + ] + assert json.loads(attrs[OUTPUT_ATTR]) == [{"role": "assistant", "content": [{"type": "text", "text": "pong"}]}] + + +def test_chat_stream_relays_chunks_untouched_and_stamps_assembled_output(): + logger, exporter = _logger() + chunks = (_chat_chunk("po"), _chat_chunk("ng"), _chat_chunk(None, finish_reason="stop")) + + relayed = _run_stream(logger, CHAT_DATA, chunks) + + assert [id(chunk) for chunk in relayed] == [id(chunk) for chunk in chunks] + output = json.loads(_root_attrs(exporter)[OUTPUT_ATTR]) + assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] + + +def test_responses_stream_stamps_output_from_the_completed_event(): + logger, exporter = _logger() + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_responses_api_response() + ) + chunks = ({"type": "response.created"}, {"type": "response.output_text.delta", "delta": "pong"}, completed) + + relayed = _run_stream(logger, {"model": "gpt-5.4-mini", "input": "ping"}, chunks) + + assert relayed == list(chunks) + output = json.loads(_root_attrs(exporter)[OUTPUT_ATTR]) + assert output[0]["content"][0]["text"] == "pong" + + +def test_anthropic_sse_stream_stamps_output_from_the_assembled_frames(): + logger, exporter = _logger() + frames = _anthropic_sse_frames() + + relayed = _run_stream( + logger, {"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "ping"}]}, frames + ) + + assert relayed == list(frames) + output = json.loads(_root_attrs(exporter)[OUTPUT_ATTR]) + assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] + + +def test_root_observation_io_survives_the_root_ending_before_the_success_callback(): + logger, exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) + logger.log_pre_api_call( + model="gpt-5.4-mini", + messages=[], + kwargs={"litellm_call_id": "call_1", "litellm_params": {"metadata": {}}}, + ) + asyncio.run( + logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + root.end() + + payload = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-5.4-mini", + "messages": CHAT_DATA["messages"], + "response": response.model_dump(), + "status": "success", + "litellm_call_id": "call_1", + "metadata": {}, + "hidden_params": {}, + } + asyncio.run( + logger.async_log_success_event( + {"standard_logging_object": payload, "litellm_params": {"metadata": {}}}, response, None, None + ) + ) + + attrs = _root_attrs(exporter) + assert INPUT_ATTR in attrs and OUTPUT_ATTR in attrs + generation = next(span for span in exporter.get_finished_spans() if span.name != LITELLM_PROXY_REQUEST_SPAN_NAME) + assert OUTPUT_ATTR in dict(generation.attributes or {}) + + +def test_root_already_ended_is_left_alone(): + logger, exporter = _logger() + root = _start_root(logger) + root.end() + + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) + + assert INPUT_ATTR not in _root_attrs(exporter) + + +def test_non_chat_call_types_do_not_stamp_input(): + logger, exporter = _logger() + root = _start_root(logger) + + asyncio.run( + logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), {"model": "e", "input": "ping"}, "aembedding") + ) + root.end() + + assert INPUT_ATTR not in _root_attrs(exporter) + + +def test_unrenderable_output_never_raises_into_the_request(): + logger, exporter = _logger() + + _run_request(logger, CHAT_DATA, "acompletion", object()) + + assert OUTPUT_ATTR not in _root_attrs(exporter) + + +@pytest.mark.parametrize( + ("capture", "mappers"), + [("no_content", ("genai", "langfuse")), ("span_only", ("genai",))], +) +def test_factory_keeps_the_base_logger_unless_langfuse_content_capture_is_on(capture, mappers): + logger, exporter = _logger(capture=capture, mappers=mappers) + + assert type(logger) is OpenTelemetryV2 + _run_request(logger, CHAT_DATA, "acompletion", ModelResponse()) + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs + + +def test_langfuse_otel_preset_builds_the_langfuse_logger(monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk") + monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") + monkeypatch.setenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "span_only") + is_otel_v2_enabled.cache_clear() + + loggers: list = [] + try: + built = _maybe_construct_otel_v2("langfuse_otel", loggers) + assert isinstance(built, LangfuseOpenTelemetryV2) + assert _maybe_construct_otel_v2("langfuse_otel", loggers) is built + finally: + is_otel_v2_enabled.cache_clear() From a677242d6f07af683b9c146133287f07b4e1459c Mon Sep 17 00:00:00 2001 From: Ali Ahmed <128928915+QuantumBreakz@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:07:13 +0500 Subject: [PATCH 089/175] fix(headroom): stop re-compressing retrieved CCR content in client tool loops (#38591) When the headroom_retrieve tool is exposed to a client that runs its own tool-execution loop (the LiteLLM MCP gateway path), the client executes the retrieve call and sends the recovered original content back as a tool result on the next turn. The guardrail then compressed that row again, and because CCR is content-addressed it collapsed back to the exact same hash it was just retrieved from. The model never saw the expansion and the agent looped. Hold tool-result rows that carry headroom_retrieve output back from the compression service, the same way the live turn and trailing tool exchange are already protected, so the expansion survives. Retrieve calls are matched by the direct headroom_retrieve name and the mcp____headroom_retrieve gateway name. Because a long gateway name is truncated past 64 chars in the OpenAI-translated view the guardrail scans, the pairing also falls back to the tool-call id read from the request's own untranslated messages, which is never truncated. Fixes #38558 --- .../guardrail_hooks/headroom/headroom.py | 120 ++++++++++++- .../guardrail_hooks/test_headroom.py | 158 ++++++++++++++++++ 2 files changed, 274 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index d8c8c2f4974..fc881a60f43 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, TypeGuard import httpx from fastapi import HTTPException from httpx import Response as HttpxResponse +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -52,6 +53,10 @@ BYPASS_HEADER: Final = "x-headroom-bypass" HEADROOM_RETRIEVE_TOOL_NAME: Final = "headroom_retrieve" _HASH_PATTERN: Final = re.compile(r"hash=([a-f0-9]{24})") _HASH_CACHE_TTL_SECONDS: Final = 15 * 60 +# Narrows the base class's bare-dict ``request_data`` at the boundary so its +# untranslated messages can be read with concrete types (values pass through by +# reference, so this is a shallow top-level reconstruction). +_REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object]) def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip @@ -116,16 +121,119 @@ def _restore_content_shapes( return restored -def _protected_indices(messages: Sequence[Mapping[str, object]]) -> frozenset[int]: +def _tool_call_name(tool_call: Mapping[str, object]) -> str | None: + function: Final = tool_call.get("function") + if not _is_str_object_dict(function): + return None + name: Final = function.get("name") + return name if isinstance(name, str) else None + + +def _is_retrieve_tool_name(name: str | None) -> bool: + """Match the retrieve tool whether called directly or via the MCP gateway. + + Server-side the tool is ``headroom_retrieve``; exposed through LiteLLM's MCP + gateway a client calls it as ``mcp____headroom_retrieve``. + """ + return name is not None and ( + name == HEADROOM_RETRIEVE_TOOL_NAME or name.endswith(f"__{HEADROOM_RETRIEVE_TOOL_NAME}") + ) + + +def _retrieve_call_ids_in_message(message: Mapping[str, object]) -> frozenset[str]: + if message.get("role") != "assistant": + return frozenset() + tool_calls: Final = message.get("tool_calls") + if not _is_object_list(tool_calls): + return frozenset() + return frozenset( + str(tool_call["id"]) + for tool_call in tool_calls + if _is_str_object_dict(tool_call) and tool_call.get("id") and _is_retrieve_tool_name(_tool_call_name(tool_call)) + ) + + +def _anthropic_tool_use_retrieve_id(block: object) -> str | None: + if not _is_str_object_dict(block) or block.get("type") != "tool_use": + return None + name: Final = block.get("name") + call_id: Final = block.get("id") + if isinstance(name, str) and call_id is not None and _is_retrieve_tool_name(name): + return str(call_id) + return None + + +def _anthropic_retrieve_ids_in_message(message: Mapping[str, object]) -> frozenset[str]: + content: Final = message.get("content") + if not _is_object_list(content): + return frozenset() + return frozenset(call_id for block in content if (call_id := _anthropic_tool_use_retrieve_id(block)) is not None) + + +def _raw_retrieve_call_ids(messages: object) -> frozenset[str]: + """Retrieve-tool call ids read from the request's own, untranslated messages. + + The guardrail otherwise scans an OpenAI-translated view where a tool name + over 64 chars is truncated to ``{prefix}_{hash}``, which drops the + ``__headroom_retrieve`` suffix a long ``mcp____`` prefix pushes past + the limit. Tool-call ids are never truncated, so pairing the tool result to + an id read from the original request keeps the match intact. Both wire + shapes are handled: OpenAI ``tool_calls`` and Anthropic ``tool_use`` blocks. + """ + if not _is_object_list(messages): + return frozenset() + return frozenset( + call_id + for message in messages + if _is_str_object_dict(message) + for call_id in _retrieve_call_ids_in_message(message) | _anthropic_retrieve_ids_in_message(message) + ) + + +def _retrieval_result_indices( + messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset() +) -> frozenset[int]: + """Indices of tool-result rows that carry ``headroom_retrieve`` output. + + When the retrieve tool is exposed to a client that runs its own tool loop + (the LiteLLM MCP gateway path), the client executes the call and sends the + recovered original content back as a tool result on the next turn. That + content is exactly what a prior compression stubbed, so compressing it again + re-derives the identical content hash: a no-op that strands the model on the + marker and loops the agent. Hold those rows back so the expansion survives. + + ``extra_retrieve_call_ids`` carries ids recovered from the untruncated + request so the pairing survives tool-name truncation (see + ``_raw_retrieve_call_ids``). + """ + retrieve_call_ids: Final = extra_retrieve_call_ids | frozenset( + call_id for message in messages for call_id in _retrieve_call_ids_in_message(message) + ) + if not retrieve_call_ids: + return frozenset() + return frozenset( + index + for index, message in enumerate(messages) + if message.get("role") in ("tool", "function") and str(message.get("tool_call_id")) in retrieve_call_ids + ) + + +def _protected_indices( + messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset() +) -> frozenset[int]: """Indices headroom must not send to the compression service. ``get_protected_indices`` is litellm's own compression policy: the system - rows, the last user row, the last assistant row. It is expanded over whole + rows, the last user row, the last assistant row. Rows carrying just-retrieved + ``headroom_retrieve`` output are added so re-compression can't collapse them + back to the marker they were expanded from. The union is expanded over whole tool exchanges the way ``compress()`` expands it, so a protected assistant tool call cannot end up answered by a marker standing in for the result the model just asked for. """ - protected: Final = frozenset(get_protected_indices(messages)) + protected: Final = frozenset(get_protected_indices(messages)) | _retrieval_result_indices( + messages, extra_retrieve_call_ids + ) return protected | frozenset( index for group in group_tool_exchanges(messages) @@ -634,7 +742,11 @@ class HeadroomGuardrail(CustomGuardrail): # /v1/compress grows a field for sending the live turn as the retrieval # query without compressing it: query-aware compression reads the newest # user message, so it is withheld here at some cost to history ranking. - protected_indices: Final = _protected_indices(messages) + # request_data is a bare dict on the base signature; narrow it before + # reading the untranslated messages so long tool names can be recovered. + raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages") + raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages) + protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids) compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices] if not compressible: return inputs diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index 1fbc975e40a..c04fb7b30ec 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -2199,6 +2199,164 @@ async def test_history_is_still_compressed(guardrail: HeadroomGuardrail): assert has_headroom_retrieve_tool(result.get("tools") or []) +# --------------------------------------------------------------------------- +# #38558: a client that runs its own tool loop (e.g. Claude Code via the MCP +# gateway) executes headroom_retrieve and echoes the recovered original content +# back as a tool result. Compressing that row re-derives the same content hash +# it was just retrieved from -- the marker returns and the agent loops. The +# retrieved row must be held back from the compression service. +# --------------------------------------------------------------------------- + +RETRIEVE_ECHO_MESSAGES = [ + {"role": "system", "content": "You are Claude Code. " + "S" * 5000}, + {"role": "user", "content": "H" * 5000}, + { + "role": "assistant", + "content": "Expanding the marker.", + "tool_calls": [ + { + "id": "hr_1", + "type": "function", + "function": { + "name": "mcp__headroom__headroom_retrieve", + "arguments": '{"hash": "b573993006976af767214fac"}', + }, + } + ], + }, + {"role": "tool", "tool_call_id": "hr_1", "content": "RETRIEVED BODY " + "R" * 5000}, + {"role": "assistant", "content": "Older answer. " + "O" * 5000}, + {"role": "user", "content": "now summarize the description"}, +] + + +@pytest.mark.asyncio +async def test_retrieved_content_is_never_recompressed(guardrail: HeadroomGuardrail): + """The tool result carrying headroom_retrieve output is held back, so it can + never collapse back to the hash it was just retrieved from.""" + wire, result = await _wire_and_result(guardrail, RETRIEVE_ECHO_MESSAGES) + + assert not any(row.get("tool_call_id") == "hr_1" for row in wire) + assert not any("RETRIEVED BODY" in json.dumps(row) for row in wire) + # Reaches the model byte-identical, so no marker stands in for the expansion. + assert result["structured_messages"][3] == RETRIEVE_ECHO_MESSAGES[3] + # Negative control: unrelated history is still compressed, not a no-op. + assert any(row.get("content") == "H" * 5000 for row in wire) + + +@pytest.mark.asyncio +async def test_retrieved_content_guard_matches_direct_tool_name(guardrail: HeadroomGuardrail): + """Server-side the tool is named headroom_retrieve (no MCP prefix); its + result must be protected the same way.""" + messages = [ + {"role": "system", "content": "sys " + "S" * 5000}, + {"role": "user", "content": "H" * 5000}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "hr_direct", + "type": "function", + "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME, "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "hr_direct", "content": "RETRIEVED BODY " + "R" * 5000}, + {"role": "assistant", "content": "Older. " + "O" * 5000}, + {"role": "user", "content": "summarize"}, + ] + wire, result = await _wire_and_result(guardrail, messages) + + assert not any(row.get("tool_call_id") == "hr_direct" for row in wire) + assert result["structured_messages"][3] == messages[3] + + +@pytest.mark.asyncio +async def test_retrieved_content_protected_when_mcp_tool_name_is_truncated(guardrail: HeadroomGuardrail): + """A long mcp____headroom_retrieve name is truncated past 64 chars in + the OpenAI-translated view the guardrail scans, dropping the suffix. The call + id read from the request's own Anthropic tool_use (never truncated) still + pairs the retrieved row so it is held back.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + truncate_tool_name, + ) + + long_name = "mcp__" + "s" * 45 + "__" + HEADROOM_RETRIEVE_TOOL_NAME + assert len(long_name) > 64 + truncated = truncate_tool_name(long_name) + assert not truncated.endswith(HEADROOM_RETRIEVE_TOOL_NAME) + + # What the guardrail scans: OpenAI-translated messages with the truncated name. + structured = [ + {"role": "system", "content": "sys " + "S" * 5000}, + {"role": "user", "content": "H" * 5000}, + { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "hr_long", "type": "function", "function": {"name": truncated, "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "hr_long", "content": "RETRIEVED BODY " + "R" * 5000}, + {"role": "assistant", "content": "Older. " + "O" * 5000}, + {"role": "user", "content": "summarize"}, + ] + # The request's own messages, untranslated: Anthropic tool_use carries the full name. + raw_messages = [ + {"role": "assistant", "content": [{"type": "tool_use", "id": "hr_long", "name": long_name, "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "hr_long", "content": "RETRIEVED BODY"}]}, + ] + + inputs = GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(structured))) + sent: dict = {} + + def _echo(**kwargs): + sent["messages"] = kwargs["json"]["messages"] + return _make_compress_response(json.loads(json.dumps(kwargs["json"]["messages"]))) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_echo): + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "claude-sonnet-4-5-20250929", "messages": raw_messages}, + input_type="request", + ) + + assert not any(row.get("tool_call_id") == "hr_long" for row in sent["messages"]) + assert result["structured_messages"][3] == structured[3] + assert any(row.get("content") == "H" * 5000 for row in sent["messages"]) + + +def test_raw_retrieve_call_ids_covers_both_shapes_and_ignores_others(): + """Retrieve ids are read from OpenAI tool_calls and Anthropic tool_use blocks; + non-retrieve calls, non-tool_use blocks, string content, and non-list inputs + yield nothing.""" + from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import _raw_retrieve_call_ids + + messages = [ + { + "role": "assistant", + "tool_calls": [ + {"id": "oa1", "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME}}, + {"id": "other", "function": {"name": "get_weather"}}, + {"id": "malformed", "function": {"name": 123}}, + {"id": "nofunc"}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "an1", "name": "mcp__hr__headroom_retrieve", "input": {}}, + {"type": "tool_use", "id": "an2", "name": "jira_get_issue", "input": {}}, + {"type": "text", "text": "noise"}, + ], + }, + {"role": "user", "content": "plain string content, not a list"}, + ] + + assert _raw_retrieve_call_ids(messages) == frozenset({"oa1", "an1"}) + assert _raw_retrieve_call_ids("not a list") == frozenset() + assert _raw_retrieve_call_ids(None) == frozenset() + + @pytest.mark.asyncio async def test_nothing_compressible_returns_inputs_untouched(guardrail: HeadroomGuardrail): """A single-turn request is all protected, so there is nothing to send and From c83b4a1d1985e79253b7d32f1454cfc96f48e01c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:10:54 -0700 Subject: [PATCH 090/175] fix(proxy-extras): honor a raised command timeout for migrate deploy and name the right knob on db push timeouts --- .../litellm_proxy_extras/prisma_toolchain.py | 13 +++-- .../litellm_proxy_extras/utils.py | 5 +- .../test_prisma_toolchain.py | 51 +++++++++++++++++-- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py b/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py index 5feb7a953b4..2283814ab35 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py +++ b/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py @@ -26,7 +26,10 @@ finish any sooner. Migrate deploy therefore runs under its own budget. All three budgets are overridable so an operator can widen them without a release: ``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install, ``LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT`` for ``prisma migrate deploy`` and -``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every other Prisma command. +``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every other Prisma command. The +per-command budget used to bound migrate deploy as well, so a deployment that +raised it above the deploy default keeps that larger budget for deploy unless +the deploy override says otherwise. """ import math @@ -102,9 +105,11 @@ def prisma_bootstrap_timeout() -> float: def prisma_migrate_deploy_timeout() -> float: """Seconds one ``prisma migrate deploy`` may run for, however many migrations are pending.""" - return _timeout_from_env( - PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT - ) + if os.getenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR) is not None: + return _timeout_from_env( + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT + ) + return max(DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT, prisma_command_timeout()) def nodeenv_cache_dir() -> Optional[Path]: diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index ab9ec1e8a3a..f6647268624 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -15,6 +15,7 @@ from litellm_proxy_extras.replica_identity import ( apply_replica_identity_full, ) from litellm_proxy_extras.prisma_toolchain import ( + PRISMA_COMMAND_TIMEOUT_ENV_VAR, PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, ensure_prisma_toolchain, prisma_command_timeout, @@ -1135,9 +1136,9 @@ class ProxyExtrasDBManager: return True except subprocess.TimeoutExpired: logger.warning( - "Attempt %s timed out. Raise %s if this database needs longer to apply its pending migrations.", + "Attempt %s timed out. Raise %s if this database needs longer to apply its schema.", attempt + 1, - PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR if use_migrate else PRISMA_COMMAND_TIMEOUT_ENV_VAR, ) time.sleep(random.randrange(5, 15)) except subprocess.CalledProcessError as e: diff --git a/tests/proxy_migration_tests/test_prisma_toolchain.py b/tests/proxy_migration_tests/test_prisma_toolchain.py index 1bd440d37b4..4c258c4d007 100644 --- a/tests/proxy_migration_tests/test_prisma_toolchain.py +++ b/tests/proxy_migration_tests/test_prisma_toolchain.py @@ -16,11 +16,13 @@ the proxy gave up after four identical timeouts. import ast import json +import logging import os import sys import time from collections.abc import Callable from pathlib import Path +from typing import Optional import pytest @@ -52,10 +54,10 @@ import time args = sys.argv[1:] cache_dir = os.environ["PRISMA_NODEENV_CACHE_DIR"] log_path = pathlib.Path(os.environ["FAKE_PRISMA_LOG"]) -earlier_deploys = sum( +earlier_same_command = sum( 1 for line in (log_path.read_text().splitlines() if log_path.exists() else []) - if json.loads(line)["args"][:2] == ["migrate", "deploy"] + if json.loads(line)["args"][:2] == args[:2] ) with log_path.open("a") as log: log.write( @@ -64,12 +66,14 @@ with log_path.open("a") as log: ) time.sleep(float(os.environ.get("FAKE_PRISMA_SLEEP", "0"))) if args[:2] == ["migrate", "deploy"]: - if earlier_deploys == 0: + if earlier_same_command == 0: time.sleep(float(os.environ.get("FAKE_PRISMA_FIRST_DEPLOY_SLEEP", "0"))) elif os.environ.get("FAKE_PRISMA_LATER_DEPLOY_STDERR"): print(os.environ["FAKE_PRISMA_LATER_DEPLOY_STDERR"], file=sys.stderr) sys.exit(1) print("No pending migrations to apply") +if args[:2] == ["db", "push"] and earlier_same_command == 0: + time.sleep(float(os.environ.get("FAKE_PRISMA_FIRST_PUSH_SLEEP", "0"))) sys.exit(0) """ @@ -269,6 +273,47 @@ def test_migrate_deploy_stops_at_its_own_timeout( assert elapsed < 30 +def test_db_push_timeout_hint_names_the_per_command_budget( + toolchain_env: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """``db push`` keeps the per-command budget, so its timeout hint has to name that variable.""" + _, log_path = toolchain_env + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, "1") + monkeypatch.setenv("FAKE_PRISMA_FIRST_PUSH_SLEEP", "3") + + with caplog.at_level(logging.WARNING, logger="litellm_proxy_extras"): + assert ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=False) is True + + assert [call["args"][:2] for call in _fake_prisma_calls(log_path)].count(["db", "push"]) == 2 + assert [record.getMessage() for record in caplog.records if "timed out" in record.getMessage()] == [ + f"Attempt 1 timed out. Raise {PRISMA_COMMAND_TIMEOUT_ENV_VAR} if this database needs longer to apply its schema." + ] + + +@pytest.mark.parametrize( + ("command_timeout", "deploy_timeout", "expected"), + [ + ("900", None, 900.0), + ("12", None, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT), + ("900", "1200", 1200.0), + ("900", "300", 300.0), + ], + ids=["raised_command_budget_carries_over", "lowered_command_budget_does_not", "override_wins_upward", "override_wins_downward"], +) +def test_migrate_deploy_budget_keeps_a_raised_command_budget( + command_timeout: str, deploy_timeout: Optional[str], expected: float, monkeypatch: pytest.MonkeyPatch +) -> None: + """Deployments that raised the per-command budget to survive a long deploy keep that budget for deploy.""" + monkeypatch.setenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, command_timeout) + if deploy_timeout is None: + monkeypatch.delenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, raising=False) + else: + monkeypatch.setenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, deploy_timeout) + + assert prisma_migrate_deploy_timeout() == expected + + @pytest.mark.parametrize( "raw", ["", "0", "-5", "not-a-number", "nan", "inf", "-inf", "1e400"], From 25987cb961567541922cef80e1433137f42c7b77 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 14:42:48 -0700 Subject: [PATCH 091/175] test(build): validate release wheel contracts --- .github/scripts/smoke_test_native_wheel.py | 68 ++++++++ .github/scripts/verify_linux_native_wheel.py | 161 +++++++++++++++++++ .github/workflows/test-rust.yml | 111 +++++++++++++ litellm-rust/crates/python-bridge/Cargo.toml | 1 + litellm-rust/crates/python-bridge/src/lib.rs | 8 + 5 files changed, 349 insertions(+) create mode 100644 .github/scripts/smoke_test_native_wheel.py create mode 100644 .github/scripts/verify_linux_native_wheel.py diff --git a/.github/scripts/smoke_test_native_wheel.py b/.github/scripts/smoke_test_native_wheel.py new file mode 100644 index 00000000000..577bb32fcf0 --- /dev/null +++ b/.github/scripts/smoke_test_native_wheel.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path +from typing import Final + +CHILD_SCRIPT: Final = """ +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +import sys + +native_path = Path(sys.argv[1]) +spec = spec_from_file_location("litellm.rust_bridge._native", native_path) +if spec is None or spec.loader is None: + raise RuntimeError("cannot create native extension import specification") +module = module_from_spec(spec) +spec.loader.exec_module(module) + +before = module.gil_stats() +if not isinstance(before.get("releases"), int): + raise AssertionError(f"unexpected gil_stats result: {before!r}") + +try: + module._panic_for_test() +except BaseException as error: + if type(error).__name__ != "PanicException": + raise AssertionError(f"expected PanicException, got {type(error).__name__}") from error +else: + raise AssertionError("Rust panic returned without raising") + +after = module.gil_stats() +if not isinstance(after.get("releases"), int): + raise AssertionError(f"native module unusable after panic: {after!r}") +""" + + +def main() -> int: + if len(sys.argv) != 2: + sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") + return 2 + + wheel: Final = Path(sys.argv[1]) + with tempfile.TemporaryDirectory() as temporary_directory, zipfile.ZipFile(wheel) as archive: + native_members: Final = tuple( + member + for member in archive.infolist() + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") + return 1 + + native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name + native_path.write_bytes(archive.read(native_members[0])) + result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False) + + if result.returncode != 0: + sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n") + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py new file mode 100644 index 00000000000..1de98ae0168 --- /dev/null +++ b/.github/scripts/verify_linux_native_wheel.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import importlib.util +import os +import re +import subprocess +import sys +import zipfile +from pathlib import Path, PurePosixPath +from typing import Final + + +def _loads_native_module(native_path: Path) -> bool: + module_spec: Final = importlib.util.spec_from_file_location("litellm.rust_bridge._native", native_path) + if module_spec is None or module_spec.loader is None: + return False + try: + native_module: Final = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(native_module) + except Exception as error: + sys.stderr.write(f"native module load failed: {error}\n") + return False + return True + + +def main() -> int: + if len(sys.argv) != 2: + sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") + return 2 + + wheel: Final = Path(sys.argv[1]) + with zipfile.ZipFile(wheel) as archive: + wheel_members: Final = archive.infolist() + native_members: Final = tuple( + member + for member in wheel_members + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") + return 1 + + unexpected_members: Final = tuple( + member.filename + for member in wheel_members + if member.filename.endswith((".pdb", ".dwp", ".rlib", ".rmeta", "Cargo.toml", "Cargo.lock")) + or any(part.endswith(".dSYM") for part in PurePosixPath(member.filename).parts) + ) + native_member: Final = native_members[0] + uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members) + native_path: Final = wheel.parent / "native" / Path(native_member.filename).name + native_path.parent.mkdir(parents=True, exist_ok=True) + native_path.write_bytes(archive.read(native_member)) + + wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) + if len(wheel_tags) != 4: + sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") + return 1 + + python_tag: Final = wheel_tags[1] + abi_tag: Final = wheel_tags[2] + platform_tag: Final = wheel_tags[3] + commit_sha: Final = os.environ.get("RELEASE_WHEEL_COMMIT_SHA", os.environ.get("GITHUB_SHA", "unknown")) + rustc_version: Final = subprocess.run( + ("rustc", "--version"), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text() + maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject) + if maturin_match is None: + sys.stderr.write("build-system does not pin an exact Maturin version\n") + return 1 + + maturin_version: Final = maturin_match.group(1) + native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100 + size_report: Final = "\n".join( + ( + "## Native wheel build report", + "", + "| Build | Value |", + "| --- | --- |", + f"| Commit | `{commit_sha}` |", + f"| Platform | `{platform_tag}` |", + f"| Python ABI | `{python_tag}-{abi_tag}` |", + f"| Rust compiler | `{rustc_version}` |", + f"| Maturin | `{maturin_version}` |", + "| Cargo profile | `release` |", + "", + "| Artifact | Size |", + "| --- | ---: |", + f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |", + f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |", + f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |", + f"| Native share | {native_percentage:.2f}% |", + "", + ) + ) + summary_path: Final = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path is None: + sys.stdout.write(size_report) + else: + Path(summary_path).write_text(size_report) + + sections: Final = subprocess.run( + ("readelf", "--sections", "--wide", native_path), + check=True, + capture_output=True, + text=True, + ).stdout + debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections) + debug_sections_absent: Final = not debug_sections + static_symbol_table_absent: Final = ".symtab" not in sections + + dynamic_symbols: Final = subprocess.run( + ("readelf", "--dyn-syms", "--wide", native_path), + check=True, + capture_output=True, + text=True, + ).stdout + extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols + native_module_loads: Final = _loads_native_module(native_path) + native_size_limit: Final = 20_000_000 + native_size_within_limit: Final = native_member.file_size <= native_size_limit + validations: Final = ( + ("Debug sections are absent", debug_sections_absent), + ("Static symbol table is absent", static_symbol_table_absent), + ("Python extension entry point is present", extension_entry_point_present), + ("Native module loads", native_module_loads), + ("Native extension does not exceed 20 MB", native_size_within_limit), + ("Wheel contents are valid", not unexpected_members), + ) + + verified_report: Final = size_report + "\n".join( + ("", "| Validation | Expected | Result |", "| --- | --- | :---: |") + + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations) + + ("",) + ) + report_path: Final = os.environ.get("RELEASE_WHEEL_REPORT") + if report_path is not None: + Path(report_path).write_text(size_report) + if summary_path is not None: + Path(summary_path).write_text(verified_report) + + if debug_sections: + sys.stderr.write(f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}\n") + if not static_symbol_table_absent: + sys.stderr.write(f"{native_member.filename} contains a static symbol table\n") + if not extension_entry_point_present: + sys.stderr.write("native extension does not export PyInit__native\n") + if not native_size_within_limit: + sys.stderr.write(f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB\n") + if unexpected_members: + sys.stderr.write(f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}\n") + + return 0 if all(passed for _, passed in validations) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 21e1bcb90c6..b05ddb81740 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -4,6 +4,10 @@ on: push: paths: - "litellm-rust/**" + - ".cargo/**" + - "pyproject.toml" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -13,6 +17,10 @@ on: - "litellm_**" paths: - "litellm-rust/**" + - ".cargo/**" + - "pyproject.toml" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" permissions: @@ -69,3 +77,106 @@ jobs: - name: Run core tests with Bedrock auth run: cargo test -p litellm-core --features bedrock-auth --locked + + release-wheel: + name: release wheel + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + pull-requests: write + env: + CARGO_TERM_COLOR: always + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Set up Rust + run: | + rustup toolchain install stable --profile minimal + rustup default stable + + - name: Cache release build + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + litellm-rust/target + key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-release- + + - name: Build release wheel + run: uv build --wheel --out-dir dist + + - name: Verify stripped native extension + env: + RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + RELEASE_WHEEL_REPORT: dist/release-wheel-report.md + run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl + + - name: Build panic contract wheel + run: >- + uv build --wheel --out-dir panic-dist + --config-setting "maturin.build-args=--features panic-test" + + - name: Smoke-test native panic unwinding + run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl + + - name: Report release wheel size on PR + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + COMMENT_MARKER: "" + REPORT_PATH: dist/release-wheel-report.md + with: + script: | + const fs = require("fs"); + const marker = process.env.COMMENT_MARKER; + const report = fs.readFileSync(process.env.REPORT_PATH, "utf8"); + const body = `${marker}\n${report}`; + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const existing = comments.find((comment) => comment.body?.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: Upload release wheel + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: litellm-release-wheel-linux-x86_64 + path: dist/*.whl + if-no-files-found: error diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index d461a483ae0..b1fdfd7677a 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -13,6 +13,7 @@ crate-type = ["cdylib"] default = ["abi3"] abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] +panic-test = [] [dependencies] litellm-core = { workspace = true, features = ["bedrock-auth"] } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f9e75f45f75..18e0b05cbb5 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -614,6 +614,12 @@ fn gil_stats(py: Python<'_>) -> PyResult> { Ok(stats.into_any().unbind()) } +#[cfg(feature = "panic-test")] +#[pyfunction] +fn _panic_for_test() { + panic!("intentional PyO3 panic smoke test"); +} + #[pymodule] fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { let py = module.py(); @@ -630,5 +636,7 @@ fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(achat_completions, module)?)?; module.add_class::()?; module.add_function(wrap_pyfunction!(gil_stats, module)?)?; + #[cfg(feature = "panic-test")] + module.add_function(wrap_pyfunction!(_panic_for_test, module)?)?; Ok(()) } From 2f362cfec2d122d4d2c14b73a988a20fe61e6629 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 14:59:03 -0700 Subject: [PATCH 092/175] fix(ci): preserve release wheel contract parity --- .github/scripts/verify_linux_native_wheel.py | 2 +- .github/workflows/test-rust.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index 1de98ae0168..b0ab34df98d 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -139,7 +139,7 @@ def main() -> int: ) report_path: Final = os.environ.get("RELEASE_WHEEL_REPORT") if report_path is not None: - Path(report_path).write_text(size_report) + Path(report_path).write_text(verified_report) if summary_path is not None: Path(summary_path).write_text(verified_report) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index b05ddb81740..1a9f36f24d2 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -132,7 +132,7 @@ jobs: - name: Build panic contract wheel run: >- uv build --wheel --out-dir panic-dist - --config-setting "maturin.build-args=--features panic-test" + --config-setting "maturin.build-args=--features panic-test,extension-module" - name: Smoke-test native panic unwinding run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl From 9dc9cd325cf5adca79d910a30c420381a28d1923 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 20:21:37 -0700 Subject: [PATCH 093/175] fix(ci): isolate release wheel reporting permissions --- .github/scripts/verify_linux_native_wheel.py | 3 - .../workflows/report-rust-release-wheel.yml | 95 +++++++++++++++++++ .github/workflows/test-rust.yml | 39 -------- 3 files changed, 95 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/report-rust-release-wheel.yml diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index b0ab34df98d..96264beb632 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -137,9 +137,6 @@ def main() -> int: + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations) + ("",) ) - report_path: Final = os.environ.get("RELEASE_WHEEL_REPORT") - if report_path is not None: - Path(report_path).write_text(verified_report) if summary_path is not None: Path(summary_path).write_text(verified_report) diff --git a/.github/workflows/report-rust-release-wheel.yml b/.github/workflows/report-rust-release-wheel.yml new file mode 100644 index 00000000000..76aefd70ef6 --- /dev/null +++ b/.github/workflows/report-rust-release-wheel.yml @@ -0,0 +1,95 @@ +name: Report LiteLLM Rust release wheel + +on: # zizmor: ignore[dangerous-triggers] reporter executes no PR code and consumes no PR artifacts or outputs + workflow_run: + workflows: + - LiteLLM Rust + types: + - completed + +permissions: {} + +jobs: + report-release-wheel: + name: report release wheel + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.path == '.github/workflows/test-rust.yml' && + github.event.workflow_run.head_repository.full_name == github.repository && + github.event.workflow_run.pull_requests[0].number != null + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write + + steps: + - name: Link release wheel report on PR + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + COMMENT_MARKER: "" + with: + script: | + const marker = process.env.COMMENT_MARKER; + const workflowRun = context.payload.workflow_run; + if ( + workflowRun.conclusion !== "success" || + workflowRun.event !== "pull_request" || + workflowRun.path !== ".github/workflows/test-rust.yml" || + workflowRun.head_repository?.full_name !== + `${context.repo.owner}/${context.repo.repo}` || + workflowRun.pull_requests?.length !== 1 + ) { + throw new Error("unexpected source workflow"); + } + const pullRequest = workflowRun.pull_requests[0]; + const pullRequestNumber = pullRequest.number; + const headSha = workflowRun.head_sha; + const runId = workflowRun.id; + if ( + !Number.isSafeInteger(pullRequestNumber) || + pullRequestNumber <= 0 || + !Number.isSafeInteger(runId) || + runId <= 0 || + !/^[0-9a-f]{40}$/.test(headSha) || + pullRequest.head?.sha !== headSha + ) { + throw new Error("invalid source workflow metadata"); + } + const runUrl = + `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + + `/actions/runs/${runId}`; + const body = [ + marker, + "## LiteLLM Rust workflow", + "", + `Workflow completed successfully for \`${headSha}\``, + "", + `[View workflow run](${runUrl})`, + ].join("\n"); + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + per_page: 100, + }); + const existing = comments.find( + (comment) => + comment.user?.login === "github-actions[bot]" && + comment.body?.startsWith(marker), + ); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body, + }); + } diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 1a9f36f24d2..b01e7fabe4a 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -84,7 +84,6 @@ jobs: timeout-minutes: 20 permissions: contents: read - pull-requests: write env: CARGO_TERM_COLOR: always @@ -126,7 +125,6 @@ jobs: - name: Verify stripped native extension env: RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - RELEASE_WHEEL_REPORT: dist/release-wheel-report.md run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl - name: Build panic contract wheel @@ -137,43 +135,6 @@ jobs: - name: Smoke-test native panic unwinding run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl - - name: Report release wheel size on PR - if: >- - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 - env: - COMMENT_MARKER: "" - REPORT_PATH: dist/release-wheel-report.md - with: - script: | - const fs = require("fs"); - const marker = process.env.COMMENT_MARKER; - const report = fs.readFileSync(process.env.REPORT_PATH, "utf8"); - const body = `${marker}\n${report}`; - const comments = await github.paginate(github.rest.issues.listComments, { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - per_page: 100, - }); - const existing = comments.find((comment) => comment.body?.includes(marker)); - if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body, - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body, - }); - } - - name: Upload release wheel uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: From e6a317e0790c575d6452f59c90e861f110b54a0c Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 20:53:17 -0700 Subject: [PATCH 094/175] fix(ci): enforce release wheel metadata contract --- .github/scripts/verify_linux_native_wheel.py | 110 ++++++++++-- .../test_verify_linux_native_wheel.py | 170 ++++++++++++++++++ 2 files changed, 266 insertions(+), 14 deletions(-) create mode 100644 tests/test_litellm/test_verify_linux_native_wheel.py diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index 96264beb632..61f176ab183 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -6,21 +6,44 @@ import re import subprocess import sys import zipfile +from email import policy +from email.parser import BytesParser +from itertools import product from pathlib import Path, PurePosixPath -from typing import Final +from types import ModuleType +from typing import Final, cast + +EXPECTED_PYTHON_TAG: Final = "cp310" +EXPECTED_ABI_TAG: Final = "abi3" +EXPECTED_PLATFORM_TAG: Final = "linux_x86_64" -def _loads_native_module(native_path: Path) -> bool: +def _dist_info_directory(member: zipfile.ZipInfo) -> str | None: + parts: Final = PurePosixPath(member.filename).parts + if not parts or not parts[0].endswith(".dist-info"): + return None + return parts[0] + + +def _wheel_metadata_tags(archive: zipfile.ZipFile, members: tuple[zipfile.ZipInfo, ...]) -> tuple[str, ...]: + if len(members) != 1: + return () + metadata: Final = BytesParser(policy=policy.default).parsebytes(archive.read(members[0])) + tags: Final = cast(list[str], metadata.get_all("Tag", [])) + return tuple(tag.strip() for tag in tags) + + +def _load_native_module(native_path: Path) -> ModuleType | None: module_spec: Final = importlib.util.spec_from_file_location("litellm.rust_bridge._native", native_path) if module_spec is None or module_spec.loader is None: - return False + return None try: native_module: Final = importlib.util.module_from_spec(module_spec) module_spec.loader.exec_module(native_module) except Exception as error: sys.stderr.write(f"native module load failed: {error}\n") - return False - return True + return None + return native_module def main() -> int: @@ -29,8 +52,39 @@ def main() -> int: return 2 wheel: Final = Path(sys.argv[1]) + wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) + if len(wheel_tags) != 4: + sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") + return 1 + + wheel_identity: Final = wheel_tags[0].split("-") + if len(wheel_identity) != 2 or wheel_identity[0] != "litellm" or not wheel_identity[1]: + sys.stderr.write(f"unexpected wheel identity: {wheel_tags[0]}\n") + return 1 + + expected_dist_info_directory: Final = f"{wheel_tags[0]}.dist-info" + expected_dist_info_directories: Final = frozenset((expected_dist_info_directory,)) + python_tag: Final = wheel_tags[1] + abi_tag: Final = wheel_tags[2] + platform_tag: Final = wheel_tags[3] + expanded_filename_tags: Final = frozenset( + "-".join(tag) for tag in product(python_tag.split("."), abi_tag.split("."), platform_tag.split(".")) + ) + with zipfile.ZipFile(wheel) as archive: wheel_members: Final = archive.infolist() + dist_info_directories: Final = frozenset( + directory for member in wheel_members if (directory := _dist_info_directory(member)) is not None + ) + required_dist_info_files: Final = ("METADATA", "RECORD", "WHEEL") + dist_info_file_counts: Final = { + filename: sum(member.filename == f"{expected_dist_info_directory}/{filename}" for member in wheel_members) + for filename in required_dist_info_files + } + wheel_metadata_members: Final = tuple( + member for member in wheel_members if member.filename == f"{expected_dist_info_directory}/WHEEL" + ) + wheel_metadata_tags: Final = _wheel_metadata_tags(archive, wheel_metadata_members) native_members: Final = tuple( member for member in wheel_members @@ -52,14 +106,10 @@ def main() -> int: native_path.parent.mkdir(parents=True, exist_ok=True) native_path.write_bytes(archive.read(native_member)) - wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) - if len(wheel_tags) != 4: - sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") - return 1 - - python_tag: Final = wheel_tags[1] - abi_tag: Final = wheel_tags[2] - platform_tag: Final = wheel_tags[3] + wheel_metadata_tags_match: Final = ( + len(wheel_metadata_tags) == len(expanded_filename_tags) + and frozenset(wheel_metadata_tags) == expanded_filename_tags + ) commit_sha: Final = os.environ.get("RELEASE_WHEEL_COMMIT_SHA", os.environ.get("GITHUB_SHA", "unknown")) rustc_version: Final = subprocess.run( ("rustc", "--version"), @@ -120,14 +170,26 @@ def main() -> int: text=True, ).stdout extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols - native_module_loads: Final = _loads_native_module(native_path) + native_module: Final = _load_native_module(native_path) + native_module_loads: Final = native_module is not None + panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") native_size_limit: Final = 20_000_000 native_size_within_limit: Final = native_member.file_size <= native_size_limit validations: Final = ( + (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), + (f"ABI tag is {EXPECTED_ABI_TAG}", abi_tag == EXPECTED_ABI_TAG), + (f"Platform tag is {EXPECTED_PLATFORM_TAG}", platform_tag == EXPECTED_PLATFORM_TAG), + ("Wheel dist-info directory matches the filename", dist_info_directories == expected_dist_info_directories), + ( + "Required dist-info files are present exactly once", + all(count == 1 for count in dist_info_file_counts.values()), + ), + ("Wheel metadata tags match the filename", wheel_metadata_tags_match), ("Debug sections are absent", debug_sections_absent), ("Static symbol table is absent", static_symbol_table_absent), ("Python extension entry point is present", extension_entry_point_present), ("Native module loads", native_module_loads), + ("Production module omits the panic test hook", panic_test_hook_absent), ("Native extension does not exceed 20 MB", native_size_within_limit), ("Wheel contents are valid", not unexpected_members), ) @@ -146,6 +208,26 @@ def main() -> int: sys.stderr.write(f"{native_member.filename} contains a static symbol table\n") if not extension_entry_point_present: sys.stderr.write("native extension does not export PyInit__native\n") + if python_tag != EXPECTED_PYTHON_TAG: + sys.stderr.write(f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}\n") + if abi_tag != EXPECTED_ABI_TAG: + sys.stderr.write(f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}\n") + if platform_tag != EXPECTED_PLATFORM_TAG: + sys.stderr.write(f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}\n") + if dist_info_directories != expected_dist_info_directories: + sys.stderr.write( + f"unexpected dist-info directories: expected {[expected_dist_info_directory]}, " + f"found {sorted(dist_info_directories)}\n" + ) + if any(count != 1 for count in dist_info_file_counts.values()): + sys.stderr.write(f"required dist-info file counts are invalid: {dist_info_file_counts}\n") + elif not wheel_metadata_tags_match: + sys.stderr.write( + f"WHEEL tags do not match filename: expected {sorted(expanded_filename_tags)}, " + f"found {sorted(wheel_metadata_tags)}\n" + ) + if native_module is not None and not panic_test_hook_absent: + sys.stderr.write("production native module exposes _panic_for_test\n") if not native_size_within_limit: sys.stderr.write(f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB\n") if unexpected_members: diff --git a/tests/test_litellm/test_verify_linux_native_wheel.py b/tests/test_litellm/test_verify_linux_native_wheel.py new file mode 100644 index 00000000000..86f5debfe3b --- /dev/null +++ b/tests/test_litellm/test_verify_linux_native_wheel.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import importlib.util +import subprocess +import sys +import zipfile +from collections.abc import Callable +from pathlib import Path +from types import ModuleType +from typing import Final, Protocol, cast + +import pytest + +_REPO_ROOT: Final = Path(__file__).resolve().parents[2] +_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "verify_linux_native_wheel.py" + + +class _VerifierModule(Protocol): + subprocess: ModuleType + _load_native_module: Callable[[Path], ModuleType | None] + main: Callable[[], int] + + +_SPEC: Final = importlib.util.spec_from_file_location("verify_linux_native_wheel", _MODULE_PATH) +assert _SPEC is not None and _SPEC.loader is not None +_LOADED_VERIFIER: Final = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _LOADED_VERIFIER +_SPEC.loader.exec_module(_LOADED_VERIFIER) +verifier: Final = cast(_VerifierModule, _LOADED_VERIFIER) + +_EXPECTED_TAG: Final = "cp310-abi3-linux_x86_64" +_NATIVE_MEMBER: Final = "litellm/rust_bridge/_native.abi3.so" +_DIST_INFO: Final = "litellm-1.100.0.dist-info" + + +def _write_wheel( + tmp_path: Path, + *, + filename_tag: str, + metadata_tags: tuple[str, ...] | None = (_EXPECTED_TAG,), + dist_info: str = _DIST_INFO, + duplicate_wheel: bool = False, +) -> Path: + wheel: Final = tmp_path / f"litellm-1.100.0-{filename_tag}.whl" + with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(_NATIVE_MEMBER, b"synthetic native extension") + archive.writestr( + f"{dist_info}/METADATA", + "Metadata-Version: 2.1\nName: litellm\nVersion: 1.100.0\n", + ) + archive.writestr( + f"{dist_info}/RECORD", + f"{_NATIVE_MEMBER},,\n{dist_info}/WHEEL,,\n", + ) + if metadata_tags is not None: + wheel_metadata: Final = ( + "Wheel-Version: 1.0\nGenerator: regression-test\nRoot-Is-Purelib: false\n" + + "".join(f"Tag: {tag}\n" for tag in metadata_tags) + ) + archive.writestr(f"{dist_info}/WHEEL", wheel_metadata) + if duplicate_wheel: + archive.writestr(f"{dist_info}/WHEEL", wheel_metadata) + return wheel + + +def _fake_subprocess_run(command: tuple[str, ...], **_: object) -> subprocess.CompletedProcess[str]: + if command == ("rustc", "--version"): + stdout = "rustc 1.98.0 (regression-test)\n" + elif "--sections" in command: + stdout = "[ 1] .text PROGBITS\n" + elif "--dyn-syms" in command: + stdout = "PyInit__native\n" + else: + raise AssertionError(f"unexpected subprocess command: {command}") + return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr="") + + +def _run_verifier( + monkeypatch: pytest.MonkeyPatch, + wheel: Path, + *, + exposes_panic: bool = False, +) -> int: + native_module: Final = ModuleType("litellm.rust_bridge._native") + if exposes_panic: + setattr(native_module, "_panic_for_test", lambda: None) + + def _fake_load_native_module(_: Path) -> ModuleType: + return native_module + + monkeypatch.setattr(verifier, "_load_native_module", _fake_load_native_module) + monkeypatch.setattr(verifier.subprocess, "run", _fake_subprocess_run) + monkeypatch.setattr(sys, "argv", [str(_MODULE_PATH), str(wheel)]) + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(wheel.parent / "summary.md")) + return verifier.main() + + +def test_accepts_expected_release_wheel_tags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + + assert _run_verifier(monkeypatch, wheel) == 0 + + +def test_rejects_cp312_version_specific_wheel(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + tag: Final = "cp312-cp312-linux_x86_64" + wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +def test_rejects_non_linux_platform_tag(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + tag: Final = "cp310-abi3-win_amd64" + wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +@pytest.mark.parametrize( + "metadata_tags", + [None, ("cp312-cp312-linux_x86_64",)], + ids=["missing", "mismatched"], +) +def test_rejects_missing_or_mismatched_wheel_metadata_tag( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + metadata_tags: tuple[str, ...] | None, +) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=metadata_tags) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +def test_rejects_wheel_metadata_from_wrong_dist_info_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + dist_info="decoy-1.0.0.dist-info", + ) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +def test_rejects_duplicate_wheel_metadata_tags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + metadata_tags=(_EXPECTED_TAG, _EXPECTED_TAG), + ) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.warns(UserWarning, match="Duplicate name"): + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + duplicate_wheel=True, + ) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +def test_rejects_production_module_exposing_panic_hook(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + + assert _run_verifier(monkeypatch, wheel, exposes_panic=True) == 1 From ef9a207ed545e740863926647af148e5195285c9 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 05:44:35 -0700 Subject: [PATCH 095/175] fix(ci): harden release wheel reporting --- .../workflows/report-rust-release-wheel.yml | 43 +++++++++++++++++-- .github/workflows/test-rust.yml | 26 ++--------- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/.github/workflows/report-rust-release-wheel.yml b/.github/workflows/report-rust-release-wheel.yml index 76aefd70ef6..1d93b56f77f 100644 --- a/.github/workflows/report-rust-release-wheel.yml +++ b/.github/workflows/report-rust-release-wheel.yml @@ -9,11 +9,14 @@ on: # zizmor: ignore[dangerous-triggers] reporter executes no PR code and consum permissions: {} +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || github.event.workflow_run.id }} + cancel-in-progress: false + jobs: report-release-wheel: name: report release wheel if: >- - github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.path == '.github/workflows/test-rust.yml' && github.event.workflow_run.head_repository.full_name == github.repository && @@ -21,7 +24,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 permissions: - issues: write + issues: write # PR comments use the issues API + pull-requests: read # Current-head validation rejects stale workflow runs steps: - name: Link release wheel report on PR @@ -32,8 +36,19 @@ jobs: script: | const marker = process.env.COMMENT_MARKER; const workflowRun = context.payload.workflow_run; + const allowedConclusions = new Set([ + "action_required", + "cancelled", + "failure", + "neutral", + "skipped", + "stale", + "startup_failure", + "success", + "timed_out", + ]); if ( - workflowRun.conclusion !== "success" || + !allowedConclusions.has(workflowRun.conclusion) || workflowRun.event !== "pull_request" || workflowRun.path !== ".github/workflows/test-rust.yml" || workflowRun.head_repository?.full_name !== @@ -59,11 +74,15 @@ jobs: const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + `/actions/runs/${runId}`; + const result = + workflowRun.conclusion === "success" + ? "successfully" + : `with \`${workflowRun.conclusion}\``; const body = [ marker, "## LiteLLM Rust workflow", "", - `Workflow completed successfully for \`${headSha}\``, + `Workflow completed ${result} for \`${headSha}\``, "", `[View workflow run](${runUrl})`, ].join("\n"); @@ -78,6 +97,22 @@ jobs: comment.user?.login === "github-actions[bot]" && comment.body?.startsWith(marker), ); + const currentPullRequest = ( + await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullRequestNumber, + }) + ).data; + if ( + currentPullRequest.state !== "open" || + currentPullRequest.head.repo?.full_name !== + `${context.repo.owner}/${context.repo.repo}` || + currentPullRequest.head.sha !== headSha + ) { + core.info("source workflow no longer matches the current pull request head"); + return; + } if (existing) { await github.rest.issues.updateComment({ owner: context.repo.owner, diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index b01e7fabe4a..77da358a2fc 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -108,25 +108,9 @@ jobs: rustup toolchain install stable --profile minimal rustup default stable - - name: Cache release build - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - litellm-rust/target - key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-release- - - name: Build release wheel run: uv build --wheel --out-dir dist - - name: Verify stripped native extension - env: - RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl - - name: Build panic contract wheel run: >- uv build --wheel --out-dir panic-dist @@ -135,9 +119,7 @@ jobs: - name: Smoke-test native panic unwinding run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl - - name: Upload release wheel - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: litellm-release-wheel-linux-x86_64 - path: dist/*.whl - if-no-files-found: error + - name: Verify stripped native extension + env: + RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl From 38150dfc2c50526664125c25c6eabf07d06e07ed Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 06:37:39 -0700 Subject: [PATCH 096/175] fix(ci): pin workflow toolchain dependencies --- .circleci/config.yml | 8 ++--- .../actions/setup-uv-with-retries/action.yml | 14 +++----- .github/workflows/test-rust.yml | 12 +++---- rust-toolchain.toml | 4 +++ .../test_circleci_rust_toolchain.py | 33 ++++++++++++++----- 5 files changed, 42 insertions(+), 29 deletions(-) create mode 100644 rust-toolchain.toml diff --git a/.circleci/config.yml b/.circleci/config.yml index 55fa9410845..dfc539fb80e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,10 +112,10 @@ commands: node --version npm --version install_rust: - description: "Install pinned rustup (1.28.2) and Rust toolchain (1.97.1) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself." + description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself." steps: - run: - name: Install Rust (rustup 1.28.2, toolchain 1.97.1) + name: Install Rust (rustup 1.28.2, toolchain 1.98.0) command: | case "$(uname -m)" in x86_64) @@ -135,7 +135,7 @@ commands: "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init" echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - chmod +x /tmp/rustup-init - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.97.1 + /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0 rm -f /tmp/rustup-init echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.cargo/bin:$PATH" @@ -300,7 +300,7 @@ jobs: if ($rustupActual -ne $rustupExpected) { throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual" } - & $rustupInit -y --profile minimal --default-toolchain stable + & $rustupInit -y --profile minimal --default-toolchain 1.98.0 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/.github/actions/setup-uv-with-retries/action.yml b/.github/actions/setup-uv-with-retries/action.yml index 1627038dc3d..98ff91f0283 100644 --- a/.github/actions/setup-uv-with-retries/action.yml +++ b/.github/actions/setup-uv-with-retries/action.yml @@ -1,11 +1,7 @@ name: "Set up uv with retries" description: >- - Install uv via astral-sh/setup-uv, retrying on transient failures. Even with - an exact pinned version, the action resolves the artifact URL by fetching - https://raw.githubusercontent.com/astral-sh/versions/main/v1/uv.ndjson in a - single request with no retry, timeout, or fallback, so one connection-level - network error ("fetch failed") fails the whole job before any test runs. - Retrying the full step covers the manifest fetch and the binary download. + Install uv via astral-sh/setup-uv, retrying the full setup step so manifest + resolution and binary downloads get fresh attempts after transient failures. inputs: version: @@ -18,7 +14,7 @@ runs: - name: Set up uv (attempt 1) id: attempt-1 continue-on-error: true - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: version: ${{ inputs.version }} @@ -31,7 +27,7 @@ runs: id: attempt-2 if: steps.attempt-1.outcome == 'failure' continue-on-error: true - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: version: ${{ inputs.version }} @@ -42,6 +38,6 @@ runs: - name: Set up uv (attempt 3) if: steps.attempt-2.outcome == 'failure' - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: version: ${{ inputs.version }} diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 77da358a2fc..aada0fcf239 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -6,6 +6,7 @@ on: - "litellm-rust/**" - ".cargo/**" - "pyproject.toml" + - "rust-toolchain.toml" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" @@ -19,6 +20,7 @@ on: - "litellm-rust/**" - ".cargo/**" - "pyproject.toml" + - "rust-toolchain.toml" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" @@ -48,9 +50,7 @@ jobs: persist-credentials: false - name: Set up Rust - run: | - rustup toolchain install stable --profile minimal --component clippy,rustfmt - rustup default stable + run: rustup toolchain install - name: Cache Cargo registry and target uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 @@ -59,7 +59,7 @@ jobs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-cargo-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-cargo-${{ hashFiles('rust-toolchain.toml', 'litellm-rust/Cargo.lock') }} restore-keys: | ${{ runner.os }}-cargo- @@ -104,9 +104,7 @@ jobs: version: "0.10.9" - name: Set up Rust - run: | - rustup toolchain install stable --profile minimal - rustup default stable + run: rustup toolchain install - name: Build release wheel run: uv build --wheel --out-dir dist diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000000..a1598ccbb34 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.98.0" +profile = "minimal" +components = ["rustfmt", "clippy"] diff --git a/tests/test_litellm/test_circleci_rust_toolchain.py b/tests/test_litellm/test_circleci_rust_toolchain.py index c35ced51e16..800ca21b95d 100644 --- a/tests/test_litellm/test_circleci_rust_toolchain.py +++ b/tests/test_litellm/test_circleci_rust_toolchain.py @@ -17,28 +17,27 @@ Two invariants are pinned here: Windows job, so the check accepts either. A new job that syncs without one falls back to the unpinned path, which is exactly the regression a static check catches at PR time and a green CI run does not. - 2. `install_rust` itself pins what it downloads: an explicit rustup version in - the URL, a verified SHA-256, and an exact toolchain version rather than a - channel name. - -The Windows job predates `install_rust` and provisions its toolchain inline, so -invariant 2 is scoped to `install_rust`; invariant 1 covers both. + 2. Both installers pin what they download: an explicit rustup version, a + verified SHA-256, and the exact toolchain in `rust-toolchain.toml`. """ from __future__ import annotations import re from pathlib import Path +from typing import Final import pytest import yaml REPO_ROOT = Path(__file__).resolve().parents[2] CONFIG = REPO_ROOT / ".circleci" / "config.yml" +TOOLCHAIN: Final = REPO_ROOT / "rust-toolchain.toml" BUILDS_WORKSPACE = re.compile(r"\buv\s+(?:sync|build)\b") RUSTUP_ARCHIVE_URL = re.compile(r"https://static\.rust-lang\.org/rustup/archive/\d+\.\d+\.\d+/") -EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?\d+\.\d+\.\d+\"?") +EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?(\d+\.\d+\.\d+)\"?") +TOOLCHAIN_CHANNEL: Final = re.compile(r'^channel = "(\d+\.\d+\.\d+)"$', re.MULTILINE) def _config() -> dict[str, object]: @@ -57,6 +56,12 @@ def _step_text(step: object) -> str: return "" +def _pinned_toolchain() -> str: + match: Final = TOOLCHAIN_CHANNEL.search(TOOLCHAIN.read_text()) + assert match is not None, "rust-toolchain.toml must pin an exact channel" + return match.group(1) + + def _without_comments(text: str) -> str: return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) @@ -142,7 +147,17 @@ def test_install_rust_verifies_the_installer_checksum(install_rust_command: str) def test_install_rust_pins_an_exact_toolchain_version(install_rust_command: str) -> None: - assert EXACT_TOOLCHAIN.search(install_rust_command), ( - "install_rust must pin an exact toolchain version (e.g. 1.97.1); a channel name like " + match: Final = EXACT_TOOLCHAIN.search(install_rust_command) + assert match is not None, ( + "install_rust must pin an exact toolchain version (e.g. 1.98.0); a channel name like " "stable/beta/nightly makes the compiler drift with whatever upstream published that day" ) + assert match.group(1) == _pinned_toolchain() + + +def test_windows_installer_matches_the_repo_toolchain() -> None: + windows_steps: Final = _step_lists()["job using_litellm_on_windows"] + windows_command: Final = "\n".join(_step_text(step) for step in windows_steps) + match: Final = EXACT_TOOLCHAIN.search(windows_command) + assert match is not None + assert match.group(1) == _pinned_toolchain() From cbb8a1784db64b8433c07d09af2a114993b717ff Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 06:49:43 -0700 Subject: [PATCH 097/175] chore(ci): extract setup-uv pin --- .github/actions/setup-uv-with-retries/action.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-uv-with-retries/action.yml b/.github/actions/setup-uv-with-retries/action.yml index 98ff91f0283..1627038dc3d 100644 --- a/.github/actions/setup-uv-with-retries/action.yml +++ b/.github/actions/setup-uv-with-retries/action.yml @@ -1,7 +1,11 @@ name: "Set up uv with retries" description: >- - Install uv via astral-sh/setup-uv, retrying the full setup step so manifest - resolution and binary downloads get fresh attempts after transient failures. + Install uv via astral-sh/setup-uv, retrying on transient failures. Even with + an exact pinned version, the action resolves the artifact URL by fetching + https://raw.githubusercontent.com/astral-sh/versions/main/v1/uv.ndjson in a + single request with no retry, timeout, or fallback, so one connection-level + network error ("fetch failed") fails the whole job before any test runs. + Retrying the full step covers the manifest fetch and the binary download. inputs: version: @@ -14,7 +18,7 @@ runs: - name: Set up uv (attempt 1) id: attempt-1 continue-on-error: true - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 with: version: ${{ inputs.version }} @@ -27,7 +31,7 @@ runs: id: attempt-2 if: steps.attempt-1.outcome == 'failure' continue-on-error: true - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 with: version: ${{ inputs.version }} @@ -38,6 +42,6 @@ runs: - name: Set up uv (attempt 3) if: steps.attempt-2.outcome == 'failure' - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 with: version: ${{ inputs.version }} From ce0c85ea691922d1e2f06bec93df10b7dc7f43ad Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 12:06:00 -0700 Subject: [PATCH 098/175] refactor(rust): colocate native wheel contract checks --- .github/workflows/test-rust.yml | 12 ++-- .../rust_bridge}/smoke_test_native_wheel.py | 0 .../rust_bridge}/verify_linux_native_wheel.py | 57 ++++++++++++---- .../test_verify_linux_native_wheel.py | 65 +++++++------------ 4 files changed, 74 insertions(+), 60 deletions(-) rename {.github/scripts => litellm/rust_bridge}/smoke_test_native_wheel.py (100%) rename {.github/scripts => litellm/rust_bridge}/verify_linux_native_wheel.py (86%) rename tests/test_litellm/{ => rust_bridge}/test_verify_linux_native_wheel.py (62%) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index aada0fcf239..271c73733a3 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -7,8 +7,8 @@ on: - ".cargo/**" - "pyproject.toml" - "rust-toolchain.toml" - - ".github/scripts/smoke_test_native_wheel.py" - - ".github/scripts/verify_linux_native_wheel.py" + - "litellm/rust_bridge/smoke_test_native_wheel.py" + - "litellm/rust_bridge/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -21,8 +21,8 @@ on: - ".cargo/**" - "pyproject.toml" - "rust-toolchain.toml" - - ".github/scripts/smoke_test_native_wheel.py" - - ".github/scripts/verify_linux_native_wheel.py" + - "litellm/rust_bridge/smoke_test_native_wheel.py" + - "litellm/rust_bridge/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" permissions: @@ -115,9 +115,9 @@ jobs: --config-setting "maturin.build-args=--features panic-test,extension-module" - name: Smoke-test native panic unwinding - run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl + run: python litellm/rust_bridge/smoke_test_native_wheel.py panic-dist/*.whl - name: Verify stripped native extension env: RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl + run: python litellm/rust_bridge/verify_linux_native_wheel.py dist/*.whl diff --git a/.github/scripts/smoke_test_native_wheel.py b/litellm/rust_bridge/smoke_test_native_wheel.py similarity index 100% rename from .github/scripts/smoke_test_native_wheel.py rename to litellm/rust_bridge/smoke_test_native_wheel.py diff --git a/.github/scripts/verify_linux_native_wheel.py b/litellm/rust_bridge/verify_linux_native_wheel.py similarity index 86% rename from .github/scripts/verify_linux_native_wheel.py rename to litellm/rust_bridge/verify_linux_native_wheel.py index 61f176ab183..783de87a8b2 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/litellm/rust_bridge/verify_linux_native_wheel.py @@ -6,18 +6,40 @@ import re import subprocess import sys import zipfile +from collections.abc import Callable, Mapping, Sequence from email import policy from email.parser import BytesParser from itertools import product from pathlib import Path, PurePosixPath from types import ModuleType -from typing import Final, cast +from typing import Final, Protocol, cast EXPECTED_PYTHON_TAG: Final = "cp310" EXPECTED_ABI_TAG: Final = "abi3" EXPECTED_PLATFORM_TAG: Final = "linux_x86_64" +class CommandRunner(Protocol): + def __call__( + self, + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +def _run_command( + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, check=check, capture_output=capture_output, text=text) + + def _dist_info_directory(member: zipfile.ZipInfo) -> str | None: parts: Final = PurePosixPath(member.filename).parts if not parts or not parts[0].endswith(".dist-info"): @@ -46,12 +68,19 @@ def _load_native_module(native_path: Path) -> ModuleType | None: return native_module -def main() -> int: - if len(sys.argv) != 2: - sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") +def main( + argv: Sequence[str] | None = None, + environment: Mapping[str, str] | None = None, + load_native_module: Callable[[Path], ModuleType | None] = _load_native_module, + run_command: CommandRunner = _run_command, +) -> int: + arguments: Final = tuple(sys.argv if argv is None else argv) + resolved_environment: Final = os.environ if environment is None else environment + if len(arguments) != 2: + sys.stderr.write(f"usage: {Path(arguments[0]).name} WHEEL\n") return 2 - wheel: Final = Path(sys.argv[1]) + wheel: Final = Path(arguments[1]) wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) if len(wheel_tags) != 4: sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") @@ -110,8 +139,10 @@ def main() -> int: len(wheel_metadata_tags) == len(expanded_filename_tags) and frozenset(wheel_metadata_tags) == expanded_filename_tags ) - commit_sha: Final = os.environ.get("RELEASE_WHEEL_COMMIT_SHA", os.environ.get("GITHUB_SHA", "unknown")) - rustc_version: Final = subprocess.run( + commit_sha: Final = resolved_environment.get( + "RELEASE_WHEEL_COMMIT_SHA", resolved_environment.get("GITHUB_SHA", "unknown") + ) + rustc_version: Final = run_command( ("rustc", "--version"), check=True, capture_output=True, @@ -147,14 +178,14 @@ def main() -> int: "", ) ) - summary_path: Final = os.environ.get("GITHUB_STEP_SUMMARY") + summary_path: Final = resolved_environment.get("GITHUB_STEP_SUMMARY") if summary_path is None: sys.stdout.write(size_report) else: Path(summary_path).write_text(size_report) - sections: Final = subprocess.run( - ("readelf", "--sections", "--wide", native_path), + sections: Final = run_command( + ("readelf", "--sections", "--wide", str(native_path)), check=True, capture_output=True, text=True, @@ -163,14 +194,14 @@ def main() -> int: debug_sections_absent: Final = not debug_sections static_symbol_table_absent: Final = ".symtab" not in sections - dynamic_symbols: Final = subprocess.run( - ("readelf", "--dyn-syms", "--wide", native_path), + dynamic_symbols: Final = run_command( + ("readelf", "--dyn-syms", "--wide", str(native_path)), check=True, capture_output=True, text=True, ).stdout extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols - native_module: Final = _load_native_module(native_path) + native_module: Final = load_native_module(native_path) native_module_loads: Final = native_module is not None panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") native_size_limit: Final = 20_000_000 diff --git a/tests/test_litellm/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py similarity index 62% rename from tests/test_litellm/test_verify_linux_native_wheel.py rename to tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py index 86f5debfe3b..a4291ce0a65 100644 --- a/tests/test_litellm/test_verify_linux_native_wheel.py +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -1,32 +1,16 @@ from __future__ import annotations -import importlib.util import subprocess -import sys import zipfile -from collections.abc import Callable from pathlib import Path from types import ModuleType -from typing import Final, Protocol, cast +from typing import Final import pytest -_REPO_ROOT: Final = Path(__file__).resolve().parents[2] -_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "verify_linux_native_wheel.py" +from litellm.rust_bridge import verify_linux_native_wheel as verifier - -class _VerifierModule(Protocol): - subprocess: ModuleType - _load_native_module: Callable[[Path], ModuleType | None] - main: Callable[[], int] - - -_SPEC: Final = importlib.util.spec_from_file_location("verify_linux_native_wheel", _MODULE_PATH) -assert _SPEC is not None and _SPEC.loader is not None -_LOADED_VERIFIER: Final = importlib.util.module_from_spec(_SPEC) -sys.modules[_SPEC.name] = _LOADED_VERIFIER -_SPEC.loader.exec_module(_LOADED_VERIFIER) -verifier: Final = cast(_VerifierModule, _LOADED_VERIFIER) +_MODULE_PATH: Final = Path(verifier.__file__) _EXPECTED_TAG: Final = "cp310-abi3-linux_x86_64" _NATIVE_MEMBER: Final = "litellm/rust_bridge/_native.abi3.so" @@ -76,7 +60,6 @@ def _fake_subprocess_run(command: tuple[str, ...], **_: object) -> subprocess.Co def _run_verifier( - monkeypatch: pytest.MonkeyPatch, wheel: Path, *, exposes_panic: bool = False, @@ -88,31 +71,33 @@ def _run_verifier( def _fake_load_native_module(_: Path) -> ModuleType: return native_module - monkeypatch.setattr(verifier, "_load_native_module", _fake_load_native_module) - monkeypatch.setattr(verifier.subprocess, "run", _fake_subprocess_run) - monkeypatch.setattr(sys, "argv", [str(_MODULE_PATH), str(wheel)]) - monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(wheel.parent / "summary.md")) - return verifier.main() + environment: Final = {"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")} + return verifier.main( + (str(_MODULE_PATH), str(wheel)), + environment, + _fake_load_native_module, + _fake_subprocess_run, + ) -def test_accepts_expected_release_wheel_tags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_accepts_expected_release_wheel_tags(tmp_path: Path) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) - assert _run_verifier(monkeypatch, wheel) == 0 + assert _run_verifier(wheel) == 0 -def test_rejects_cp312_version_specific_wheel(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_cp312_version_specific_wheel(tmp_path: Path) -> None: tag: Final = "cp312-cp312-linux_x86_64" wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 -def test_rejects_non_linux_platform_tag(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None: tag: Final = "cp310-abi3-win_amd64" wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 @pytest.mark.parametrize( @@ -122,17 +107,15 @@ def test_rejects_non_linux_platform_tag(tmp_path: Path, monkeypatch: pytest.Monk ) def test_rejects_missing_or_mismatched_wheel_metadata_tag( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, metadata_tags: tuple[str, ...] | None, ) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=metadata_tags) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 def test_rejects_wheel_metadata_from_wrong_dist_info_directory( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: wheel: Final = _write_wheel( tmp_path, @@ -140,20 +123,20 @@ def test_rejects_wheel_metadata_from_wrong_dist_info_directory( dist_info="decoy-1.0.0.dist-info", ) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 -def test_rejects_duplicate_wheel_metadata_tags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_duplicate_wheel_metadata_tags(tmp_path: Path) -> None: wheel: Final = _write_wheel( tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=(_EXPECTED_TAG, _EXPECTED_TAG), ) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 -def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path) -> None: with pytest.warns(UserWarning, match="Duplicate name"): wheel: Final = _write_wheel( tmp_path, @@ -161,10 +144,10 @@ def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path, monkeypatch: pyte duplicate_wheel=True, ) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 -def test_rejects_production_module_exposing_panic_hook(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_production_module_exposing_panic_hook(tmp_path: Path) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) - assert _run_verifier(monkeypatch, wheel, exposes_panic=True) == 1 + assert _run_verifier(wheel, exposes_panic=True) == 1 From 814204e21f87c65234d5167cff92c92d357a5587 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 16:06:39 -0700 Subject: [PATCH 099/175] fix(rust): satisfy native wheel verifier lint --- .../rust_bridge/verify_linux_native_wheel.py | 93 +++++++++++-------- .../test_verify_linux_native_wheel.py | 44 +++++---- 2 files changed, 80 insertions(+), 57 deletions(-) diff --git a/litellm/rust_bridge/verify_linux_native_wheel.py b/litellm/rust_bridge/verify_linux_native_wheel.py index 783de87a8b2..899e2a211c0 100644 --- a/litellm/rust_bridge/verify_linux_native_wheel.py +++ b/litellm/rust_bridge/verify_linux_native_wheel.py @@ -7,12 +7,10 @@ import subprocess import sys import zipfile from collections.abc import Callable, Mapping, Sequence -from email import policy -from email.parser import BytesParser from itertools import product from pathlib import Path, PurePosixPath -from types import ModuleType -from typing import Final, Protocol, cast +from types import MappingProxyType, ModuleType +from typing import Final, Protocol EXPECTED_PYTHON_TAG: Final = "cp310" EXPECTED_ABI_TAG: Final = "abi3" @@ -50,9 +48,8 @@ def _dist_info_directory(member: zipfile.ZipInfo) -> str | None: def _wheel_metadata_tags(archive: zipfile.ZipFile, members: tuple[zipfile.ZipInfo, ...]) -> tuple[str, ...]: if len(members) != 1: return () - metadata: Final = BytesParser(policy=policy.default).parsebytes(archive.read(members[0])) - tags: Final = cast(list[str], metadata.get_all("Tag", [])) - return tuple(tag.strip() for tag in tags) + lines: Final = archive.read(members[0]).splitlines() + return tuple(line.removeprefix(b"Tag:").strip().decode("ascii") for line in lines if line.startswith(b"Tag:")) def _load_native_module(native_path: Path) -> ModuleType | None: @@ -62,7 +59,7 @@ def _load_native_module(native_path: Path) -> ModuleType | None: try: native_module: Final = importlib.util.module_from_spec(module_spec) module_spec.loader.exec_module(native_module) - except Exception as error: + except Exception as error: # noqa: BLE001 # native module initialization can raise arbitrary exceptions sys.stderr.write(f"native module load failed: {error}\n") return None return native_module @@ -106,10 +103,14 @@ def main( directory for member in wheel_members if (directory := _dist_info_directory(member)) is not None ) required_dist_info_files: Final = ("METADATA", "RECORD", "WHEEL") - dist_info_file_counts: Final = { - filename: sum(member.filename == f"{expected_dist_info_directory}/{filename}" for member in wheel_members) - for filename in required_dist_info_files - } + dist_info_file_counts: Final = MappingProxyType( + { + filename: sum( + member.filename == f"{expected_dist_info_directory}/{filename}" for member in wheel_members + ) + for filename in required_dist_info_files + } + ) wheel_metadata_members: Final = tuple( member for member in wheel_members if member.filename == f"{expected_dist_info_directory}/WHEEL" ) @@ -233,36 +234,46 @@ def main( if summary_path is not None: Path(summary_path).write_text(verified_report) - if debug_sections: - sys.stderr.write(f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}\n") - if not static_symbol_table_absent: - sys.stderr.write(f"{native_member.filename} contains a static symbol table\n") - if not extension_entry_point_present: - sys.stderr.write("native extension does not export PyInit__native\n") - if python_tag != EXPECTED_PYTHON_TAG: - sys.stderr.write(f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}\n") - if abi_tag != EXPECTED_ABI_TAG: - sys.stderr.write(f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}\n") - if platform_tag != EXPECTED_PLATFORM_TAG: - sys.stderr.write(f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}\n") - if dist_info_directories != expected_dist_info_directories: - sys.stderr.write( - f"unexpected dist-info directories: expected {[expected_dist_info_directory]}, " - f"found {sorted(dist_info_directories)}\n" + invalid_dist_info_files: Final = any(count != 1 for count in dist_info_file_counts.values()) + validation_errors: Final = tuple( + message + for failed, message in ( + (bool(debug_sections), f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}"), + (not static_symbol_table_absent, f"{native_member.filename} contains a static symbol table"), + (not extension_entry_point_present, "native extension does not export PyInit__native"), + ( + python_tag != EXPECTED_PYTHON_TAG, + f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}", + ), + (abi_tag != EXPECTED_ABI_TAG, f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}"), + ( + platform_tag != EXPECTED_PLATFORM_TAG, + f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}", + ), + ( + dist_info_directories != expected_dist_info_directories, + f"unexpected dist-info directories: expected {expected_dist_info_directory}, " + f"found {', '.join(sorted(dist_info_directories))}", + ), + (invalid_dist_info_files, f"required dist-info file counts are invalid: {dist_info_file_counts}"), + ( + not invalid_dist_info_files and not wheel_metadata_tags_match, + f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, " + f"found {', '.join(sorted(wheel_metadata_tags))}", + ), + ( + native_module is not None and not panic_test_hook_absent, + "production native module exposes _panic_for_test", + ), + ( + not native_size_within_limit, + f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB", + ), + (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"), ) - if any(count != 1 for count in dist_info_file_counts.values()): - sys.stderr.write(f"required dist-info file counts are invalid: {dist_info_file_counts}\n") - elif not wheel_metadata_tags_match: - sys.stderr.write( - f"WHEEL tags do not match filename: expected {sorted(expanded_filename_tags)}, " - f"found {sorted(wheel_metadata_tags)}\n" - ) - if native_module is not None and not panic_test_hook_absent: - sys.stderr.write("production native module exposes _panic_for_test\n") - if not native_size_within_limit: - sys.stderr.write(f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB\n") - if unexpected_members: - sys.stderr.write(f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}\n") + if failed + ) + sys.stderr.write("".join(f"{message}\n" for message in validation_errors)) return 0 if all(passed for _, passed in validations) else 1 diff --git a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py index a4291ce0a65..8d0082dddc1 100644 --- a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -3,7 +3,7 @@ from __future__ import annotations import subprocess import zipfile from pathlib import Path -from types import ModuleType +from types import MappingProxyType, ModuleType from typing import Final import pytest @@ -47,16 +47,26 @@ def _write_wheel( return wheel -def _fake_subprocess_run(command: tuple[str, ...], **_: object) -> subprocess.CompletedProcess[str]: +def _fake_subprocess_run( + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + assert check and capture_output and text if command == ("rustc", "--version"): - stdout = "rustc 1.98.0 (regression-test)\n" - elif "--sections" in command: - stdout = "[ 1] .text PROGBITS\n" - elif "--dyn-syms" in command: - stdout = "PyInit__native\n" - else: - raise AssertionError(f"unexpected subprocess command: {command}") - return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr="") + return subprocess.CompletedProcess(command, 0, stdout="rustc 1.98.0 (regression-test)\n", stderr="") + if "--sections" in command: + return subprocess.CompletedProcess(command, 0, stdout="[ 1] .text PROGBITS\n", stderr="") + if "--dyn-syms" in command: + return subprocess.CompletedProcess(command, 0, stdout="PyInit__native\n", stderr="") + raise AssertionError(f"unexpected subprocess command: {command}") + + +class _NativeModuleWithPanicHook(ModuleType): + def _panic_for_test(self) -> None: + return None def _run_verifier( @@ -64,14 +74,16 @@ def _run_verifier( *, exposes_panic: bool = False, ) -> int: - native_module: Final = ModuleType("litellm.rust_bridge._native") - if exposes_panic: - setattr(native_module, "_panic_for_test", lambda: None) + native_module: Final = ( + _NativeModuleWithPanicHook("litellm.rust_bridge._native") + if exposes_panic + else ModuleType("litellm.rust_bridge._native") + ) def _fake_load_native_module(_: Path) -> ModuleType: return native_module - environment: Final = {"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")} + environment: Final = MappingProxyType({"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")}) return verifier.main( (str(_MODULE_PATH), str(wheel)), environment, @@ -102,8 +114,8 @@ def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None: @pytest.mark.parametrize( "metadata_tags", - [None, ("cp312-cp312-linux_x86_64",)], - ids=["missing", "mismatched"], + (None, ("cp312-cp312-linux_x86_64",)), + ids=("missing", "mismatched"), ) def test_rejects_missing_or_mismatched_wheel_metadata_tag( tmp_path: Path, From 90eadac40927315831bcc7d7ae6de57f155acbd8 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 06:55:19 -0700 Subject: [PATCH 100/175] test(build): keep wheel checks outside package --- .../scripts}/smoke_test_native_wheel.py | 0 .../scripts}/verify_linux_native_wheel.py | 0 .github/workflows/test-rust.yml | 12 +++--- .../test_verify_linux_native_wheel.py | 38 +++++++++++++++++-- 4 files changed, 41 insertions(+), 9 deletions(-) rename {litellm/rust_bridge => .github/scripts}/smoke_test_native_wheel.py (100%) rename {litellm/rust_bridge => .github/scripts}/verify_linux_native_wheel.py (100%) diff --git a/litellm/rust_bridge/smoke_test_native_wheel.py b/.github/scripts/smoke_test_native_wheel.py similarity index 100% rename from litellm/rust_bridge/smoke_test_native_wheel.py rename to .github/scripts/smoke_test_native_wheel.py diff --git a/litellm/rust_bridge/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py similarity index 100% rename from litellm/rust_bridge/verify_linux_native_wheel.py rename to .github/scripts/verify_linux_native_wheel.py diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 271c73733a3..aada0fcf239 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -7,8 +7,8 @@ on: - ".cargo/**" - "pyproject.toml" - "rust-toolchain.toml" - - "litellm/rust_bridge/smoke_test_native_wheel.py" - - "litellm/rust_bridge/verify_linux_native_wheel.py" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -21,8 +21,8 @@ on: - ".cargo/**" - "pyproject.toml" - "rust-toolchain.toml" - - "litellm/rust_bridge/smoke_test_native_wheel.py" - - "litellm/rust_bridge/verify_linux_native_wheel.py" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" permissions: @@ -115,9 +115,9 @@ jobs: --config-setting "maturin.build-args=--features panic-test,extension-module" - name: Smoke-test native panic unwinding - run: python litellm/rust_bridge/smoke_test_native_wheel.py panic-dist/*.whl + run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl - name: Verify stripped native extension env: RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - run: python litellm/rust_bridge/verify_linux_native_wheel.py dist/*.whl + run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl diff --git a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py index 8d0082dddc1..e449d4392d8 100644 --- a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -1,16 +1,48 @@ from __future__ import annotations +import importlib.util import subprocess +import sys import zipfile +from collections.abc import Callable, Mapping, Sequence from pathlib import Path from types import MappingProxyType, ModuleType -from typing import Final +from typing import Final, Protocol, cast import pytest -from litellm.rust_bridge import verify_linux_native_wheel as verifier -_MODULE_PATH: Final = Path(verifier.__file__) +class _CommandRunner(Protocol): + def __call__( + self, + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +class _VerifierModule(Protocol): + main: Callable[ + [ + Sequence[str] | None, + Mapping[str, str] | None, + Callable[[Path], ModuleType | None], + _CommandRunner, + ], + int, + ] + + +_REPO_ROOT: Final = Path(__file__).resolve().parents[3] +_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "verify_linux_native_wheel.py" +_SPEC: Final = importlib.util.spec_from_file_location("verify_linux_native_wheel", _MODULE_PATH) +assert _SPEC is not None and _SPEC.loader is not None +_LOADED_VERIFIER: Final = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _LOADED_VERIFIER +_SPEC.loader.exec_module(_LOADED_VERIFIER) +verifier: Final = cast(_VerifierModule, _LOADED_VERIFIER) _EXPECTED_TAG: Final = "cp310-abi3-linux_x86_64" _NATIVE_MEMBER: Final = "litellm/rust_bridge/_native.abi3.so" From 9de4e84feb128f2248253d9f52ae13049f2c88a5 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 15:07:57 -0700 Subject: [PATCH 101/175] refactor(rust): extract domain-neutral Python interop --- litellm-rust/AGENTS.md | 9 +- litellm-rust/CLAUDE.md | 9 +- litellm-rust/Cargo.lock | 109 +++++++++++++++++- litellm-rust/Cargo.toml | 3 + litellm-rust/README.md | 8 +- .../PROVIDER_CODING_STANDARDS.md | 2 +- litellm-rust/crates/ai-gateway/README.md | 7 +- .../core/tests/workspace_crate_allowlist.rs | 16 ++- litellm-rust/crates/python-bridge/AGENTS.md | 2 +- litellm-rust/crates/python-bridge/CLAUDE.md | 5 +- litellm-rust/crates/python-bridge/Cargo.toml | 3 +- .../python-bridge/benches/serialization.rs | 11 +- litellm-rust/crates/python-bridge/src/gil.rs | 32 ----- litellm-rust/crates/python-bridge/src/lib.rs | 16 +-- .../python-bridge/tests/marshal_boundary.rs | 11 +- litellm-rust/crates/python-interop/AGENTS.md | 1 + litellm-rust/crates/python-interop/Cargo.toml | 15 +++ litellm-rust/crates/python-interop/src/gil.rs | 21 ++++ litellm-rust/crates/python-interop/src/lib.rs | 5 + .../src/marshal.rs | 0 .../crates/python-interop/tests/interop.rs | 44 +++++++ 21 files changed, 247 insertions(+), 82 deletions(-) delete mode 100644 litellm-rust/crates/python-bridge/src/gil.rs create mode 100644 litellm-rust/crates/python-interop/AGENTS.md create mode 100644 litellm-rust/crates/python-interop/Cargo.toml create mode 100644 litellm-rust/crates/python-interop/src/gil.rs create mode 100644 litellm-rust/crates/python-interop/src/lib.rs rename litellm-rust/crates/{python-bridge => python-interop}/src/marshal.rs (100%) create mode 100644 litellm-rust/crates/python-interop/tests/interop.rs diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index 36a5ad5a8f4..b8b6291283d 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -1,6 +1,6 @@ # AGENTS.md -litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are MODULES inside the layers. +litellm-rust has four crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers. ## Crates @@ -8,9 +8,10 @@ litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes ( |-------|------| | litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. | | litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. ## Where a route lives @@ -28,7 +29,7 @@ core/src/messages/ Handlers never live in `ai-gateway`. `ocr`, `audio_transcription`, and `realtime` are still hosted there from before this rule; they move to `core` as they are touched. -Adding a crate: default to a MODULE. New crate ONLY on a real trigger — separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these. +Adding a crate: default to a module. A new crate requires a real trigger: separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these. Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional. diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md index fe6ceedbb86..3dcf1853efc 100644 --- a/litellm-rust/CLAUDE.md +++ b/litellm-rust/CLAUDE.md @@ -21,12 +21,13 @@ variants of it. The test for a good abstraction is that adding the next provider is a few declarative lines, not a new file of duplicated flow. Only diverge from the base when behavior is genuinely different, and say so explicitly in the PR. -## Crates (exactly three — see AGENTS.md) +## Crates (see AGENTS.md) `litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call. `litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and -`litellm-python-bridge` exposes it to the Python SDK. A crate is a **layer**, not -a route — add modules, not crates. +`litellm-python-bridge` exposes it to the Python SDK. `litellm-python-interop` +holds domain-neutral PyO3 primitives shared by Python-facing Rust code. A crate +is a layer or shared foundation, not a route; add modules, not crates. ## Core Boundary @@ -175,7 +176,7 @@ cd litellm-rust cargo fmt --check # the ai-gateway binary + server code is behind the `server` feature cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings -cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings +cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings cargo test --workspace ``` diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 4388e561026..dd41cf0e84b 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -919,6 +919,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.33" @@ -972,6 +978,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "h2" version = "0.3.27" @@ -1432,14 +1444,24 @@ dependencies = [ "criterion", "litellm-ai-gateway", "litellm-core", + "litellm-python-interop", "pyo3", "pyo3-async-runtimes", - "pythonize", - "serde", "serde_json", "tokio", ] +[[package]] +name = "litellm-python-interop" +version = "0.1.0" +dependencies = [ + "pyo3", + "pythonize", + "rstest", + "serde", + "serde_json", +] + [[package]] name = "litemap" version = "0.8.2" @@ -1627,6 +1649,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -1899,6 +1930,12 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.28" @@ -1956,6 +1993,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.119", + "unicode-ident", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -2488,6 +2554,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + [[package]] name = "tower" version = "0.5.3" @@ -2903,6 +2999,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index c17a0605fc7..c447d915abe 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/core", "crates/ai-gateway", + "crates/python-interop", "crates/python-bridge", ] resolver = "2" @@ -15,12 +16,14 @@ repository = "https://github.com/BerriAI/litellm" [workspace.dependencies] litellm-core = { path = "crates/core" } litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false } +litellm-python-interop = { path = "crates/python-interop" } axum = "0.7" pyo3 = "0.29.0" pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } pythonize = "0.29.0" rand = "0.8" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] } +rstest = "0.26.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10" diff --git a/litellm-rust/README.md b/litellm-rust/README.md index bcccf93300b..a0d79c6f0a5 100644 --- a/litellm-rust/README.md +++ b/litellm-rust/README.md @@ -26,9 +26,10 @@ coverage and production evidence. |-------|------| | litellm-core | The SDK. Per-route entrypoints (`messages::messages()`), types, provider transforms (modules under `providers/`), provider resolution, auth, the provider HTTP call, and the router. | | litellm-ai-gateway | The axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. ## Layout @@ -38,7 +39,8 @@ crates/ src/messages/ mod.rs (entrypoint), types, transformation, prepare, handler, client src/providers/anthropic/messages/transformation.rs ai-gateway/ Axum server + WebSocket hosts; calls core entrypoints. - python-bridge/ PyO3 bridge for Python LiteLLM. + python-interop/ Domain-neutral PyO3 conversion and GIL primitives. + python-bridge/ PyO3 API adapter for Python LiteLLM. ``` The folder shape follows the Python provider tree: diff --git a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md index a1860d8a9c9..4a689cb9579 100644 --- a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md +++ b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md @@ -54,6 +54,6 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages` cd litellm-rust cargo fmt --check cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings - cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings + cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings cargo test --workspace ``` diff --git a/litellm-rust/crates/ai-gateway/README.md b/litellm-rust/crates/ai-gateway/README.md index 7a6c620ee84..5cbb47220be 100644 --- a/litellm-rust/crates/ai-gateway/README.md +++ b/litellm-rust/crates/ai-gateway/README.md @@ -6,15 +6,16 @@ dials OpenAI upstream, and splices the two sockets frame-by-frame. ## Crates -`litellm-rust` is exactly three crates (a crate is a **layer**, not a route): +`litellm-rust` has four crates. A crate is a layer or shared foundation, not a route: | Crate | Role | |-------|------| | litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. | | litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. - **Client endpoint:** `wss:///v1/realtime?model=` (WebSocket) - **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset) diff --git a/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs b/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs index 656ba033b62..8a8a5ea263a 100644 --- a/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs +++ b/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs @@ -1,7 +1,8 @@ -//! Enforcement: the litellm-rust workspace has exactly three crates. +//! Enforcement: the litellm-rust workspace has exactly four crates. //! -//! `core` (pure translation), `ai-gateway` (routes + all network I/O), and -//! `python-bridge` (the PyO3 cdylib). Adding or removing a crate must be a +//! `core` (the Rust SDK), `ai-gateway` (the HTTP/WebSocket host), +//! `python-interop` (domain-neutral PyO3 primitives), and `python-bridge` (the +//! PyO3 cdylib). Adding or removing a crate must be a //! deliberate act: this test fails until the allowlist here is updated, forcing //! whoever changes the crate set to justify the new crate per the rule that a //! crate is a layer needing independent compilation / its own deps / a separate @@ -16,10 +17,15 @@ use std::path::{Path, PathBuf}; /// The one true crate set. Update BOTH this and `litellm-rust/AGENTS.md` when the /// workspace legitimately gains or loses a crate. -const EXPECTED_MEMBERS: &[&str] = &["crates/core", "crates/ai-gateway", "crates/python-bridge"]; +const EXPECTED_MEMBERS: &[&str] = &[ + "crates/core", + "crates/ai-gateway", + "crates/python-interop", + "crates/python-bridge", +]; /// The crate subdirectory names that must exist under `crates/`. -const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-bridge"]; +const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-interop", "python-bridge"]; const MISMATCH: &str = "litellm-rust crate set changed — update this allowlist AND litellm-rust/AGENTS.md, and justify the crate per the rule (crate = layer needing independent compilation / its own deps / a separate artifact)."; diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index ad3cddfa5fd..42282ca4da4 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -1,3 +1,3 @@ -litellm-python-bridge is the PyO3 cdylib that exposes Rust to the litellm Python SDK — a thin adapter (Python objects → Rust calls → Python results) over the litellm-core route entrypoints (e.g. `litellm_core::messages::messages`). +litellm-python-bridge is the PyO3 cdylib that exposes LiteLLM Rust APIs to the Python SDK. Keep API registration, domain dependency wiring, request assembly, and Python exception mapping here. Put domain-neutral Python/Serde conversion and GIL primitives in litellm-python-interop. Keep it thin: no business logic, no transforms, no I/O orchestration — just marshal in/out and call the core entrypoint. diff --git a/litellm-rust/crates/python-bridge/CLAUDE.md b/litellm-rust/crates/python-bridge/CLAUDE.md index 3ce8b8c639a..d25ae5a8130 100644 --- a/litellm-rust/crates/python-bridge/CLAUDE.md +++ b/litellm-rust/crates/python-bridge/CLAUDE.md @@ -5,8 +5,9 @@ Rules for `litellm-rust/crates/python-bridge`. ## Responsibility `python-bridge` is the PyO3 boundary between Python LiteLLM and Rust transforms. -Keep this crate thin. It adapts Python objects to Rust payloads and returns -Python-compatible dictionaries. +Keep this crate thin. It exposes LiteLLM Rust APIs, assembles domain requests, +maps domain errors to Python exceptions, and delegates generic conversion and +GIL handling to `litellm-python-interop`. ## Bridge Shape diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index b1fdfd7677a..498003de149 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -18,10 +18,9 @@ panic-test = [] [dependencies] litellm-core = { workspace = true, features = ["bedrock-auth"] } litellm-ai-gateway = { workspace = true, default-features = false } +litellm-python-interop.workspace = true pyo3.workspace = true pyo3-async-runtimes.workspace = true -pythonize.workspace = true -serde.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/python-bridge/benches/serialization.rs b/litellm-rust/crates/python-bridge/benches/serialization.rs index 8a90cf667d0..0b9436d0cb7 100644 --- a/litellm-rust/crates/python-bridge/benches/serialization.rs +++ b/litellm-rust/crates/python-bridge/benches/serialization.rs @@ -2,6 +2,7 @@ use std::hint::black_box; use std::time::Duration; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use litellm_python_interop::{from_py, to_py}; use pyo3::prelude::*; use pyo3::types::PyDict; use serde_json::{Value, json}; @@ -25,7 +26,7 @@ fn former_json_roundtrip_from_py(py: Python<'_>, value: &Bound<'_, PyAny>) -> Va } fn pythonize_from_py(value: &Bound<'_, PyAny>) -> Value { - pythonize::depythonize(value).expect("payload should depythonize") + from_py(value).expect("payload should depythonize") } fn former_json_roundtrip_to_py(py: Python<'_>, value: &Value) -> Py { @@ -37,12 +38,10 @@ fn former_json_roundtrip_to_py(py: Python<'_>, value: &Value) -> Py { } fn pythonize_to_py(py: Python<'_>, value: &Value) -> Py { - pythonize::pythonize(py, value) - .expect("response should pythonize") - .unbind() + to_py(py, value).expect("response should pythonize") } -fn serialization(c: &mut Criterion) { +fn bridge_serialization(c: &mut Criterion) { Python::initialize(); Python::attach(|py| { for &(label, payload_bytes) in PAYLOAD_SIZES { @@ -98,6 +97,6 @@ criterion_group! { .sample_size(20) .warm_up_time(Duration::from_secs(1)) .measurement_time(Duration::from_secs(4)); - targets = serialization + targets = bridge_serialization } criterion_main!(benches); diff --git a/litellm-rust/crates/python-bridge/src/gil.rs b/litellm-rust/crates/python-bridge/src/gil.rs deleted file mode 100644 index e887c8ec1e3..00000000000 --- a/litellm-rust/crates/python-bridge/src/gil.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! GIL accounting. -//! -//! A single chokepoint for releasing the GIL around blocking work. Every -//! blocking call in the bridge goes through [`release_gil`] instead of calling -//! `Python::detach` directly, so the release count stays accurate and we -//! have one place to extend later (timing histograms, per-call labels, etc.). - -use std::sync::atomic::{AtomicU64, Ordering}; - -use pyo3::prelude::*; - -/// Number of times the bridge has released the GIL since process start. -static GIL_RELEASES: AtomicU64 = AtomicU64::new(0); - -/// Release the GIL around `f`, recording the release. -/// -/// `f` must not touch any Python state — that is what makes releasing the GIL -/// safe. Returning the value back to Python re-acquires the GIL at the call -/// site, after `f` has finished. -pub fn release_gil(py: Python<'_>, f: F) -> T -where - F: FnOnce() -> T + Send, - T: Send, -{ - GIL_RELEASES.fetch_add(1, Ordering::Relaxed); - py.detach(f) -} - -/// Total GIL releases performed by the bridge so far. -pub fn release_count() -> u64 { - GIL_RELEASES.load(Ordering::Relaxed) -} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 18e0b05cbb5..746e0770f9b 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -13,16 +13,12 @@ use litellm_core::chat_completions::{ use litellm_core::error::CoreError; use litellm_core::messages::messages as run_messages; use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; +use litellm_python_interop::{from_py, release_count, release_gil, to_py}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyAny, PyDict}; use serde_json::{Map, Value}; -mod gil; -mod marshal; - -use marshal::{from_py, to_py}; - pyo3::create_exception!( _native, RustBridgeDeclined, @@ -230,7 +226,7 @@ fn ocr( timeout_seconds, )?; - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_ocr(OcrRequest { model: &model, document, @@ -318,7 +314,7 @@ fn transcription( }; let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; let timeout = optional_timeout(timeout_seconds); - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_audio_transcription( AudioTranscriptionRequest { model: &model, @@ -419,7 +415,7 @@ fn messages( let (body, extra_headers, timeout) = marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_messages(MessagesRequest { model: &model, body, @@ -546,7 +542,7 @@ fn chat_completions( timeout_seconds, )?; - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_chat_completions( ChatCompletionsRequest { model: &model, @@ -610,7 +606,7 @@ fn achat_completions( #[pyfunction] fn gil_stats(py: Python<'_>) -> PyResult> { let stats = PyDict::new(py); - stats.set_item("releases", gil::release_count())?; + stats.set_item("releases", release_count())?; Ok(stats.into_any().unbind()) } diff --git a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs index 6a6ede22e85..d397d20b9fd 100644 --- a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs +++ b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs @@ -1,7 +1,7 @@ use std::fs; use std::path::{Path, PathBuf}; -const DISALLOWED_OUTSIDE_MARSHAL: &[&str] = &[ +const DISALLOWED_OUTSIDE_INTEROP: &[&str] = &[ "py.import(\"json\")", "pythonize::", "serde_json::to_string", @@ -33,18 +33,15 @@ fn rust_sources(directory: &Path) -> Vec { } #[test] -fn serialization_is_centralized_in_marshal_module() { +fn serialization_uses_the_interop_boundary() { let root = source_root(); for path in rust_sources(&root) { - if path == root.join("marshal.rs") { - continue; - } let source = fs::read_to_string(&path).expect("bridge source should be readable"); - for disallowed in DISALLOWED_OUTSIDE_MARSHAL { + for disallowed in DISALLOWED_OUTSIDE_INTEROP { assert!( !source.contains(disallowed), - "{} bypasses the typed marshal module with `{disallowed}`", + "{} bypasses litellm-python-interop with `{disallowed}`", path.display() ); } diff --git a/litellm-rust/crates/python-interop/AGENTS.md b/litellm-rust/crates/python-interop/AGENTS.md new file mode 100644 index 00000000000..d1d61e5dfa0 --- /dev/null +++ b/litellm-rust/crates/python-interop/AGENTS.md @@ -0,0 +1 @@ +litellm-python-interop is the domain-neutral PyO3 foundation. Keep generic Python/Serde conversion and interpreter primitives here. Do not add LiteLLM domain crates, route types, API registration, or cdylib build features. diff --git a/litellm-rust/crates/python-interop/Cargo.toml b/litellm-rust/crates/python-interop/Cargo.toml new file mode 100644 index 00000000000..9da6af6e2e2 --- /dev/null +++ b/litellm-rust/crates/python-interop/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "litellm-python-interop" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +pyo3.workspace = true +pythonize.workspace = true +serde.workspace = true + +[dev-dependencies] +rstest.workspace = true +serde_json.workspace = true diff --git a/litellm-rust/crates/python-interop/src/gil.rs b/litellm-rust/crates/python-interop/src/gil.rs new file mode 100644 index 00000000000..04b966a6002 --- /dev/null +++ b/litellm-rust/crates/python-interop/src/gil.rs @@ -0,0 +1,21 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use pyo3::prelude::*; + +static GIL_RELEASES: AtomicU64 = AtomicU64::new(0); + +/// Runs work detached from the interpreter and records the release. +/// +/// `f` must not access Python state while the interpreter is detached. +pub fn release_gil(py: Python<'_>, f: F) -> T +where + F: FnOnce() -> T + Send, + T: Send, +{ + GIL_RELEASES.fetch_add(1, Ordering::Relaxed); + py.detach(f) +} + +pub fn release_count() -> u64 { + GIL_RELEASES.load(Ordering::Relaxed) +} diff --git a/litellm-rust/crates/python-interop/src/lib.rs b/litellm-rust/crates/python-interop/src/lib.rs new file mode 100644 index 00000000000..df2bd260fdb --- /dev/null +++ b/litellm-rust/crates/python-interop/src/lib.rs @@ -0,0 +1,5 @@ +mod gil; +mod marshal; + +pub use gil::{release_count, release_gil}; +pub use marshal::{from_py, to_py}; diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-interop/src/marshal.rs similarity index 100% rename from litellm-rust/crates/python-bridge/src/marshal.rs rename to litellm-rust/crates/python-interop/src/marshal.rs diff --git a/litellm-rust/crates/python-interop/tests/interop.rs b/litellm-rust/crates/python-interop/tests/interop.rs new file mode 100644 index 00000000000..9c456dcb938 --- /dev/null +++ b/litellm-rust/crates/python-interop/tests/interop.rs @@ -0,0 +1,44 @@ +use pyo3::Python; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; + +use litellm_python_interop::{from_py, release_count, release_gil, to_py}; + +struct InitializedPython; + +impl InitializedPython { + fn attach(&self, f: F) -> R + where + F: for<'py> FnOnce(Python<'py>) -> R, + { + Python::attach(f) + } +} + +#[fixture] +#[once] +fn initialized_python() -> InitializedPython { + Python::initialize(); + InitializedPython +} + +#[rstest] +fn serde_values_round_trip_through_python(#[from(initialized_python)] python: &InitializedPython) { + python.attach(|py| { + let expected = json!({"model": "test", "items": [1, true, null]}); + let python_value = to_py(py, &expected).expect("value should convert to Python"); + let actual: Value = + from_py(python_value.bind(py)).expect("Python value should convert to serde"); + + assert_eq!(actual, expected); + }); +} + +#[rstest] +fn release_gil_runs_work_and_records_it(#[from(initialized_python)] python: &InitializedPython) { + let before = release_count(); + let result = python.attach(|py| release_gil(py, || 42)); + + assert_eq!(result, 42); + assert_eq!(release_count(), before + 1); +} From 518a2a70f159c1e118d1ea0d387c7da8ba87d642 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 05:47:01 -0700 Subject: [PATCH 102/175] refactor(rust): standardize the core Error type --- .../src/audio_transcription/common_utils.rs | 10 +-- .../src/audio_transcription/handler.rs | 31 ++++---- .../src/audio_transcription/hooks.rs | 58 +++++++------- .../ai-gateway/src/audio_transcription/mod.rs | 4 +- .../crates/ai-gateway/src/io/realtime.rs | 43 +++++------ .../crates/ai-gateway/src/io/realtime_pool.rs | 4 +- .../crates/ai-gateway/src/io/responses_ws.rs | 76 +++++++++---------- .../crates/ai-gateway/src/ocr/common_utils.rs | 73 +++++++++--------- .../crates/ai-gateway/src/ocr/handler.rs | 15 ++-- .../crates/ai-gateway/src/ocr/hooks.rs | 65 ++++++++-------- litellm-rust/crates/ai-gateway/src/ocr/mod.rs | 4 +- .../crates/ai-gateway/src/ocr/tests.rs | 8 +- .../crates/ai-gateway/src/python/config.rs | 12 ++- .../ai-gateway/src/routes/messages/mod.rs | 38 +++++----- .../ai-gateway/src/routes/messages/service.rs | 16 ++-- .../ai-gateway/src/routes/realtime/service.rs | 11 ++- .../src/routes/responses/service.rs | 12 +-- .../src/audio_transcription/transformation.rs | 11 ++- .../crates/core/src/call_lifecycle/mod.rs | 34 ++++----- .../core/src/chat_completions/common_utils.rs | 7 +- .../core/src/chat_completions/handler.rs | 30 ++++---- .../crates/core/src/chat_completions/mod.rs | 5 +- .../core/src/chat_completions/prepare.rs | 21 +++-- .../crates/core/src/chat_completions/tests.rs | 40 +++++----- .../src/chat_completions/transformation.rs | 11 ++- litellm-rust/crates/core/src/error.rs | 8 +- litellm-rust/crates/core/src/http_utils.rs | 8 +- litellm-rust/crates/core/src/lib.rs | 2 +- .../crates/core/src/messages/common_utils.rs | 7 +- .../crates/core/src/messages/handler.rs | 25 +++--- litellm-rust/crates/core/src/messages/mod.rs | 7 +- .../crates/core/src/messages/prepare.rs | 12 +-- .../crates/core/src/messages/tests.rs | 10 +-- .../core/src/messages/transformation.rs | 11 ++- .../crates/core/src/ocr/transformation.rs | 11 ++- .../anthropic/chat_completions/tests.rs | 16 ++-- .../chat_completions/transformation.rs | 25 +++--- .../anthropic/messages/transformation.rs | 12 +-- .../azure_ai/messages/transformation.rs | 22 +++--- .../providers/azure_ai/ocr/transformation.rs | 62 +++++++-------- .../providers/bedrock/audio_transcription.rs | 20 ++--- .../core/src/providers/bedrock/aws_base.rs | 38 +++++----- .../bedrock/chat_completions/tests.rs | 14 ++-- .../chat_completions/transformation.rs | 23 +++--- .../providers/mistral/ocr/transformation.rs | 28 +++---- .../openai/realtime/transformation.rs | 10 +-- .../openai/responses/transformation.rs | 6 +- .../providers/vertex_ai/ocr/transformation.rs | 50 ++++++------ .../core/src/realtime/transformation.rs | 6 +- .../core/src/responses/instrumentation.rs | 8 +- .../crates/core/src/responses/websocket.rs | 6 +- litellm-rust/crates/python-bridge/src/lib.rs | 36 ++++----- 52 files changed, 544 insertions(+), 578 deletions(-) diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs index 270d5c2d97a..140bc8aeea8 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs @@ -1,10 +1,8 @@ -use std::collections::BTreeMap; - -use litellm_core::CoreResult; use litellm_core::audio_transcription::transformation::AudioTranscriptionProviderConfig; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG; use serde_json::{Map, Value}; +use std::collections::BTreeMap; pub(super) fn audio_transcription_provider_config( provider: &str, @@ -17,7 +15,7 @@ pub(super) fn audio_transcription_provider_config( pub(super) fn string_headers( headers: Option>, -) -> CoreResult> { +) -> Result, Error> { headers .unwrap_or_default() .into_iter() @@ -26,7 +24,7 @@ pub(super) fn string_headers( .as_str() .map(|value| (key.clone(), value.to_string())) .ok_or_else(|| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "audio transcription extra_headers.{key} must be a string" )) }) diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs index 33c13550f58..1bdd4ae72a2 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs @@ -1,11 +1,9 @@ -use std::time::SystemTime; - -use litellm_core::CoreResult; use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::providers::bedrock::audio_transcription::aws_auth_config; use litellm_core::providers::bedrock::aws_base::{resolve_credentials, sign_bedrock_post}; use serde_json::Value; +use std::time::SystemTime; use super::common_utils::truncate_error_body; use super::types::ProviderAudioTranscriptionRequest; @@ -13,10 +11,9 @@ use crate::client::http_client; pub(crate) async fn execute_audio_transcription_provider_call( request: ProviderAudioTranscriptionRequest, -) -> CoreResult { - let body = serde_json::to_vec(&request.body).map_err(|error| { - CoreError::InvalidRequest(format!("invalid audio request body: {error}")) - })?; +) -> Result { + let body = serde_json::to_vec(&request.body) + .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?; let mut request_builder = http_client().post(&request.url).body(body.clone()); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -27,21 +24,20 @@ pub(crate) async fn execute_audio_transcription_provider_call( let response = request_builder .send() .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; let status = response.status(); let text = response .text() .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } - let response_json: Value = serde_json::from_str(&text).map_err(|error| { - CoreError::InvalidResponse(format!("invalid audio response JSON: {error}")) - })?; + let response_json: Value = serde_json::from_str(&text) + .map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?; Ok(request .config .transform_transcription_response(&request.model, response_json)? @@ -51,14 +47,13 @@ pub(crate) async fn execute_audio_transcription_provider_call( pub(crate) async fn sign_request( request: &ProviderAudioTranscriptionRequest, optional_params: &serde_json::Map, -) -> CoreResult { +) -> Result { let env_lookup = environment_lookup; let auth = request .config .auth_strategy(&request.model, optional_params, &env_lookup)?; - let body = serde_json::to_vec(&request.body).map_err(|error| { - CoreError::InvalidRequest(format!("invalid audio request body: {error}")) - })?; + let body = serde_json::to_vec(&request.body) + .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?; let mut headers = super::common_utils::string_headers(None)?; headers.insert("Content-Type".to_string(), "application/json".to_string()); headers.extend(request.upstream_headers.iter().cloned()); diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs index 0c9faeda6e7..5e1240de759 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs @@ -1,11 +1,9 @@ -use std::future::Future; -use std::pin::Pin; - -use litellm_core::CoreResult; use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth; use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use serde_json::{Map, Value, json}; +use std::future::Future; +use std::pin::Pin; use super::common_utils::{audio_transcription_provider_config, has_header, string_headers}; use super::handler::sign_request; @@ -26,7 +24,7 @@ pub(crate) struct AudioTranscriptionLifecycleHooks { request_metadata: RequestMetadata, } -type AudioFuture<'a, T> = Pin> + Send + 'a>>; +type AudioFuture<'a, T> = Pin> + Send + 'a>>; type AudioLogFuture<'a> = Pin + Send + 'a>>; impl AudioTranscriptionLifecycleHooks { @@ -45,7 +43,7 @@ impl AudioTranscriptionLifecycleHooks { async fn run_pre_call_guardrails( &self, request: PreparedAudioTranscriptionRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -63,17 +61,17 @@ impl AudioTranscriptionLifecycleHooks { .await .map_err(guardrail_error_to_core_error)?; let Value::Object(mut data) = guardrail_request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription pre_call guardrail must return an object".to_string(), )); }; let audio = data.remove("audio").ok_or_else(|| { - CoreError::InvalidRequest("audio transcription guardrail removed audio".to_string()) + Error::InvalidRequest("audio transcription guardrail removed audio".to_string()) })?; let optional_params = match data.remove("optional_params") { Some(Value::Object(value)) => value, Some(_) => { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription optional_params must be an object".to_string(), )); } @@ -89,9 +87,9 @@ impl AudioTranscriptionLifecycleHooks { async fn prepare_provider_request( &self, request: PreparedAudioTranscriptionRequest, - ) -> CoreResult { + ) -> Result { let config = audio_transcription_provider_config(&request.custom_llm_provider) - .ok_or_else(|| CoreError::InvalidProvider(request.custom_llm_provider.clone()))?; + .ok_or_else(|| Error::InvalidProvider(request.custom_llm_provider.clone()))?; let env_lookup = super::handler::environment_lookup; let headers = string_headers(request.extra_headers)?; let url = config.complete_url( @@ -135,7 +133,7 @@ impl AudioTranscriptionLifecycleHooks { async fn run_during_call_guardrails( &self, request: ProviderAudioTranscriptionRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -153,12 +151,12 @@ impl AudioTranscriptionLifecycleHooks { .await .map_err(guardrail_error_to_core_error)?; let Value::Object(mut data) = guardrail_request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription during_call guardrail must return an object".to_string(), )); }; let body = data.remove("body").ok_or_else(|| { - CoreError::InvalidRequest("audio transcription guardrail removed body".to_string()) + Error::InvalidRequest("audio transcription guardrail removed body".to_string()) })?; Ok(ProviderAudioTranscriptionRequest { body, ..request }) } @@ -241,7 +239,7 @@ impl CallLifecycleHooks( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -281,22 +279,22 @@ fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { } } -fn guardrail_error_to_core_error(error: GuardrailError) -> CoreError { - CoreError::InvalidRequest(format!("{}: {}", error.kind, error.message)) +fn guardrail_error_to_core_error(error: GuardrailError) -> Error { + Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) } -fn core_error_kind(error: &CoreError) -> &'static str { +fn core_error_kind(error: &Error) -> &'static str { match error { - CoreError::Auth(_) => "AuthError", - CoreError::InvalidProvider(_) => "InvalidProvider", - CoreError::InvalidRequest(_) => "InvalidRequest", - CoreError::InvalidType { .. } => "InvalidType", - CoreError::MissingField(_) => "MissingField", - CoreError::Http { .. } => "HttpError", - CoreError::InvalidResponse(_) => "InvalidResponse", - CoreError::Network(_) => "NetworkError", - CoreError::Connect(_) => "ConnectError", - CoreError::Routing(_) => "RoutingError", - CoreError::Unsupported(_) => "UnsupportedRequest", + Error::Auth(_) => "AuthError", + Error::InvalidProvider(_) => "InvalidProvider", + Error::InvalidRequest(_) => "InvalidRequest", + Error::InvalidType { .. } => "InvalidType", + Error::MissingField(_) => "MissingField", + Error::Http { .. } => "HttpError", + Error::InvalidResponse(_) => "InvalidResponse", + Error::Network(_) => "NetworkError", + Error::Connect(_) => "ConnectError", + Error::Routing(_) => "RoutingError", + Error::Unsupported(_) => "UnsupportedRequest", } } diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs index 5d33d912c40..3983846d7b6 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs @@ -1,4 +1,4 @@ -use litellm_core::CoreResult; +use litellm_core::Error; use litellm_core::call_lifecycle::CallLifecycle; use serde_json::Value; @@ -13,7 +13,7 @@ pub use types::AudioTranscriptionRequest; use handler::execute_audio_transcription_provider_call; use prepare::{PreparedAudioTranscriptionCall, prepare_audio_transcription_call}; -pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> CoreResult { +pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { let PreparedAudioTranscriptionCall { request, hooks } = prepare_audio_transcription_call(request); CallLifecycle::default() diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime.rs b/litellm-rust/crates/ai-gateway/src/io/realtime.rs index 845e7bf9527..662f7328982 100644 --- a/litellm-rust/crates/ai-gateway/src/io/realtime.rs +++ b/litellm-rust/crates/ai-gateway/src/io/realtime.rs @@ -15,8 +15,7 @@ use std::time::Duration; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{Sink, SinkExt, Stream, StreamExt}; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::realtime::transformation::RealtimeProviderConfig; use litellm_core::realtime::types::RealtimeEvent; use tokio::net::TcpStream; @@ -48,7 +47,7 @@ pub(crate) type UpstreamRx = SplitStream; /// Resolve the OpenAI API key from the explicit param or the environment. /// /// Blank/whitespace values are treated as absent (guard at resolution time). -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { +pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) @@ -58,7 +57,7 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { .ok() .filter(|key| !key.trim().is_empty()) }) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } /// Open the upstream WebSocket to OpenAI for `(model, api_key, api_base)`. @@ -70,24 +69,24 @@ pub(crate) async fn dial_upstream( model: &str, api_key: &str, api_base: Option<&str>, -) -> CoreResult { +) -> Result { let url = OPENAI_REALTIME_CONFIG.complete_url(api_base, model); let mut request = url .as_str() .into_client_request() - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; // GA realtime: only Authorization. The legacy OpenAI-Beta header triggers // beta_api_shape_disabled, so we do not send it. request.headers_mut().insert( AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|err| CoreError::Auth(err.to_string()))?, + .map_err(|err| Error::Auth(err.to_string()))?, ); let (upstream, _response) = connect_async(request) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; Ok(upstream) } @@ -96,22 +95,22 @@ pub(crate) async fn dial_upstream( /// Used by the pool to pre-read OpenAI's unprompted `session.created`. Returns an /// error on a non-text frame, a closed socket, or undecodable JSON so the pool can /// discard a misbehaving socket rather than warm it. -pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> CoreResult { +pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> Result { loop { let message = upstream_rx .next() .await - .ok_or_else(|| CoreError::Network("upstream closed before first event".to_string()))? - .map_err(|err| CoreError::Network(err.to_string()))?; + .ok_or_else(|| Error::Network("upstream closed before first event".to_string()))? + .map_err(|err| Error::Network(err.to_string()))?; match message { Message::Text(text) => { return serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(err.to_string())); + .map_err(|err| Error::InvalidResponse(err.to_string())); } // Ignore protocol frames (ping/pong) while waiting for the first event. Message::Ping(_) | Message::Pong(_) => continue, Message::Close(_) => { - return Err(CoreError::Network( + return Err(Error::Network( "upstream closed before first event".to_string(), )); } @@ -139,7 +138,7 @@ pub(crate) async fn splice( mut observe: impl FnMut(&RealtimeEvent) + Send, mut client_in: In, mut client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -154,7 +153,7 @@ where client_out .send(outbound) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } @@ -175,26 +174,26 @@ where // inflate its own spend log. Logging observes upstream events only. for outbound in config.transform_realtime_request(&event, model)?.events { let payload = serde_json::to_string(&outbound) - .map_err(|err| CoreError::InvalidResponse(err.to_string()))?; + .map_err(|err| Error::InvalidResponse(err.to_string()))?; upstream_tx .send(Message::Text(payload)) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } // upstream -> client upstream_message = upstream_rx.next() => { let Some(message) = upstream_message else { break }; // upstream closed - match message.map_err(|err| CoreError::Network(err.to_string()))? { + match message.map_err(|err| Error::Network(err.to_string()))? { Message::Text(text) => { let event: RealtimeEvent = serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(err.to_string()))?; + .map_err(|err| Error::InvalidResponse(err.to_string()))?; observe(&event); for outbound in config.transform_realtime_response(&event, model)?.events { client_out .send(outbound) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } Message::Close(_) => break, @@ -225,7 +224,7 @@ pub async fn realtime( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -258,7 +257,7 @@ pub async fn realtime_warm( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs b/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs index 4a1a3cd1166..49e9c459a88 100644 --- a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs +++ b/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs @@ -28,7 +28,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use futures_util::StreamExt; -use litellm_core::CoreResult; +use litellm_core::Error; use litellm_core::realtime::types::RealtimeEvent; use crate::io::realtime::{ @@ -438,7 +438,7 @@ impl RealtimePool { /// /// `key.api_key` is already resolved (non-blank). The first frame OpenAI sends /// unprompted is `session.created`; we buffer exactly that and read nothing more. -async fn warm_one(key: &UpstreamKey) -> CoreResult { +async fn warm_one(key: &UpstreamKey) -> Result { let upstream: UpstreamWs = dial_upstream(&key.model, &key.api_key, key.api_base.as_deref()).await?; let (tx, mut rx) = upstream.split(); diff --git a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs index 9b51019f4bc..0b01747b1a5 100644 --- a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs +++ b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs @@ -4,10 +4,10 @@ use std::time::Duration; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{Sink, SinkExt, Stream, StreamExt}; +use litellm_core::Error; use litellm_core::providers::openai::responses::transformation::OPENAI_RESPONSES_WS_CONFIG; use litellm_core::responses::types::ResponsesWsEvent; use litellm_core::responses::websocket::ResponsesWebSocketProviderConfig; -use litellm_core::{CoreError, CoreResult}; use tokio::net::TcpStream; use tokio::sync::Mutex; use tokio_tungstenite::tungstenite::Message; @@ -37,51 +37,49 @@ impl ResponsesWebSocketConnection { url: &str, headers: &HashMap, timeout: Option, - ) -> CoreResult { + ) -> Result { let mut request = url .into_client_request() - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; for (name, value) in headers { let header_name = name .parse::() - .map_err(|error| CoreError::InvalidRequest(error.to_string()))?; + .map_err(|error| Error::InvalidRequest(error.to_string()))?; let header_value = HeaderValue::from_str(value) - .map_err(|error| CoreError::InvalidRequest(error.to_string()))?; + .map_err(|error| Error::InvalidRequest(error.to_string()))?; request.headers_mut().insert(header_name, header_value); } let connect = connect_async(request); let result = match timeout { Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| { - CoreError::Network("Responses WebSocket connection timed out".to_string()) + Error::Network("Responses WebSocket connection timed out".to_string()) })?, None => connect.await, }; let (socket, _) = result.map_err(|error| match error { - tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http { + tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http { status: response.status().as_u16(), body: String::new(), }, - other => CoreError::Network(other.to_string()), + other => Error::Network(other.to_string()), })?; Ok(Self { socket: Arc::new(Mutex::new(Some(socket))), }) } - pub async fn send_text(&self, text: String) -> CoreResult<()> { + pub async fn send_text(&self, text: String) -> Result<(), Error> { let mut socket = self.socket.lock().await; let Some(socket) = socket.as_mut() else { - return Err(CoreError::Network( - "Responses WebSocket is closed".to_string(), - )); + return Err(Error::Network("Responses WebSocket is closed".to_string())); }; socket .send(Message::Text(text)) .await - .map_err(|error| CoreError::Network(error.to_string())) + .map_err(|error| Error::Network(error.to_string())) } - pub async fn recv_text(&self) -> CoreResult> { + pub async fn recv_text(&self) -> Result, Error> { let mut socket_guard = self.socket.lock().await; let Some(socket) = socket_guard.as_mut() else { return Ok(None); @@ -90,27 +88,27 @@ impl ResponsesWebSocketConnection { Some(Ok(Message::Text(text))) => Ok(Some(text)), Some(Ok(Message::Binary(bytes))) => String::from_utf8(bytes.to_vec()) .map(Some) - .map_err(|error| CoreError::InvalidResponse(error.to_string())), + .map_err(|error| Error::InvalidResponse(error.to_string())), Some(Ok(Message::Close(_))) | None => Ok(None), Some(Ok(_)) => Ok(None), - Some(Err(error)) => Err(CoreError::Network(error.to_string())), + Some(Err(error)) => Err(Error::Network(error.to_string())), } } - pub async fn close(&self) -> CoreResult<()> { + pub async fn close(&self) -> Result<(), Error> { let mut socket = self.socket.lock().await; if let Some(socket) = socket.as_mut() { socket .close(None) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } *socket = None; Ok(()) } } -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { +pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { api_key .map(str::trim) .filter(|value| !value.is_empty()) @@ -120,38 +118,38 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { .ok() .filter(|value| !value.trim().is_empty()) }) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } async fn dial_upstream( model: &str, api_key: &str, api_base: Option<&str>, -) -> CoreResult { +) -> Result { let url = OPENAI_RESPONSES_WS_CONFIG.complete_websocket_url(api_base, model); let mut request = url .as_str() .into_client_request() - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; request.headers_mut().insert( AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|error| CoreError::Auth(error.to_string()))?, + .map_err(|error| Error::Auth(error.to_string()))?, ); let result = tokio::time::timeout( Duration::from_secs(DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS), connect_async(request), ) .await - .map_err(|_| CoreError::Network("Responses WebSocket connection timed out".to_string()))?; + .map_err(|_| Error::Network("Responses WebSocket connection timed out".to_string()))?; result .map(|(socket, _)| socket) .map_err(|error| match error { - tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http { + tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http { status: response.status().as_u16(), body: String::new(), }, - other => CoreError::Network(other.to_string()), + other => Error::Network(other.to_string()), }) } @@ -166,7 +164,7 @@ impl ResponsesWebSocketStreaming { observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, - ) -> CoreResult<()> + ) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -193,7 +191,7 @@ pub(crate) async fn splice( mut observe: impl FnMut(&ResponsesWsEvent) + Send, mut client_in: In, mut client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -210,18 +208,18 @@ where .events { let payload = serde_json::to_string(&outbound) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; upstream_tx.send(Message::Text(payload)) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } message = upstream_rx.next() => { let Some(message) = message else { break }; - match message.map_err(|error| CoreError::Network(error.to_string()))? { + match message.map_err(|error| Error::Network(error.to_string()))? { Message::Text(text) => { let event = serde_json::from_str::(&text) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; observe(&event); for outbound in OPENAI_RESPONSES_WS_CONFIG .transform_ws_response(&event, model)? @@ -229,7 +227,7 @@ where { client_out.send(outbound) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } Message::Close(_) => break, @@ -252,7 +250,7 @@ pub async fn async_responses_websocket( mut observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -267,11 +265,11 @@ where .events { let payload = serde_json::to_string(&outbound) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; upstream_tx .send(Message::Text(payload)) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } ResponsesWebSocketStreaming::bidirectional_forward( @@ -296,7 +294,7 @@ pub async fn responses_ws( observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -514,7 +512,7 @@ mod tests { ) .await .expect_err("status error"); - assert!(matches!(error, CoreError::Http { status: 401, .. })); + assert!(matches!(error, Error::Http { status: 401, .. })); server.await.expect("server task"); } @@ -543,7 +541,7 @@ mod tests { ) .await .expect_err("status error"); - assert!(matches!(error, CoreError::Http { status: 500, .. })); + assert!(matches!(error, Error::Http { status: 500, .. })); server.await.expect("server task"); } } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs b/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs index 9bc2818b6e7..e0ce165dc93 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs @@ -3,8 +3,7 @@ use std::time::{Duration, Instant}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrProviderConfig; use reqwest::Url; use serde_json::{Map, Value}; @@ -56,7 +55,7 @@ fn is_azure_document_intelligence_model(model: &str) -> bool { pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { extra_headers .unwrap_or_default() .into_iter() @@ -65,7 +64,7 @@ pub(super) fn string_headers( .as_str() .map(|value| (key.clone(), value.to_string())) .ok_or_else(|| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "OCR extra_headers.{key} must be a string, got {}", litellm_core::error::json_type_name(&value) )) @@ -80,7 +79,7 @@ pub(super) fn has_header(headers: &[(String, String)], name: &str) -> bool { .any(|(key, _)| key.eq_ignore_ascii_case(name)) } -fn document_url_field(document: &Value) -> CoreResult> { +fn document_url_field(document: &Value) -> Result, Error> { let Some(object) = document.as_object() else { return Ok(None); }; @@ -138,13 +137,13 @@ fn is_blocked_ip(ip: IpAddr) -> bool { } } -fn blocked_url_error(url: &Url) -> CoreError { - CoreError::InvalidRequest(format!( +fn blocked_url_error(url: &Url) -> Error { + Error::InvalidRequest(format!( "OCR document URL rejected by SSRF protection: {url}" )) } -async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { +async fn validate_safe_fetch_url(url: &Url) -> Result<(), Error> { if !matches!(url.scheme(), "http" | "https") { return Err(blocked_url_error(url)); } @@ -162,7 +161,7 @@ async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { .ok_or_else(|| blocked_url_error(url))?; let addresses = tokio::net::lookup_host((host, port)) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let mut saw_address = false; for address in addresses { saw_address = true; @@ -176,25 +175,25 @@ async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { Ok(()) } -fn redirect_location(response: &reqwest::Response, url: &Url) -> CoreResult { +fn redirect_location(response: &reqwest::Response, url: &Url) -> Result { let location = response .headers() .get(reqwest::header::LOCATION) .and_then(|value| value.to_str().ok()) .ok_or_else(|| { - CoreError::InvalidResponse("OCR document redirect missing Location header".to_string()) + Error::InvalidResponse("OCR document redirect missing Location header".to_string()) })?; url.join(location) - .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR document redirect: {err}"))) + .map_err(|err| Error::InvalidResponse(format!("invalid OCR document redirect: {err}"))) } -async fn safe_get_document_url(url: &str) -> CoreResult<(Url, reqwest::Response)> { +async fn safe_get_document_url(url: &str) -> Result<(Url, reqwest::Response), Error> { let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .build() - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let mut current_url = Url::parse(url) - .map_err(|err| CoreError::InvalidRequest(format!("invalid OCR document URL: {err}")))?; + .map_err(|err| Error::InvalidRequest(format!("invalid OCR document URL: {err}")))?; for _ in 0..MAX_SAFE_FETCH_REDIRECTS { validate_safe_fetch_url(¤t_url).await?; @@ -202,28 +201,28 @@ async fn safe_get_document_url(url: &str) -> CoreResult<(Url, reqwest::Response) .get(current_url.clone()) .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !response.status().is_redirection() { return Ok((current_url, response)); } current_url = redirect_location(&response, ¤t_url)?; } - Err(CoreError::InvalidRequest( + Err(Error::InvalidRequest( "Too many redirects while fetching OCR document URL".to_string(), )) } -fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> CoreResult<()> { +fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Result<(), Error> { if max_bytes == 0 { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "OCR document URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}" ))); } if content_length > max_bytes { let size_mb = content_length as f64 / (1024.0 * 1024.0); let max_size_mb = max_bytes as f64 / (1024.0 * 1024.0); - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "OCR document size ({size_mb:.2}MB) exceeds maximum allowed size ({max_size_mb:.2}MB). url={url}" ))); } @@ -233,7 +232,7 @@ fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Core async fn read_response_with_limit( mut response: reqwest::Response, url: &Url, -) -> CoreResult> { +) -> Result, Error> { let max_bytes = max_document_download_bytes(); if let Some(content_length) = response.content_length() { enforce_download_size(content_length, max_bytes, url)?; @@ -246,7 +245,7 @@ async fn read_response_with_limit( while let Some(chunk) = response .chunk() .await - .map_err(|err| CoreError::Network(err.to_string()))? + .map_err(|err| Error::Network(err.to_string()))? { bytes_downloaded += chunk.len() as u64; enforce_download_size(bytes_downloaded, max_bytes, url)?; @@ -255,7 +254,7 @@ async fn read_response_with_limit( Ok(bytes) } -pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreResult { +pub(super) async fn convert_document_url_to_data_uri(document: Value) -> Result { let Some((field, url)) = document_url_field(&document)? else { return Ok(document); }; @@ -267,7 +266,7 @@ pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreRes let status = response.status(); if !status.is_success() { let body = response.text().await.unwrap_or_default(); - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&body), }); @@ -290,7 +289,7 @@ pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreRes let mut transformed = document .as_object() .cloned() - .ok_or_else(|| CoreError::InvalidRequest("OCR document must be an object".to_string()))?; + .ok_or_else(|| Error::InvalidRequest("OCR document must be an object".to_string()))?; transformed.insert(field.to_string(), Value::String(data_uri)); Ok(Value::Object(transformed)) } @@ -316,11 +315,11 @@ fn retry_after_secs(response: &reqwest::Response) -> u64 { .unwrap_or(2) } -fn operation_status(response_json: &Value) -> CoreResult<&str> { +fn operation_status(response_json: &Value) -> Result<&str, Error> { let status = response_json .get("status") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("status"))?; + .ok_or(Error::MissingField("status"))?; match status { "succeeded" => Ok("succeeded"), "running" | "notStarted" => Ok("running"), @@ -330,11 +329,11 @@ fn operation_status(response_json: &Value) -> CoreResult<&str> { .and_then(|error| error.get("message")) .and_then(Value::as_str) .unwrap_or("Unknown error"); - Err(CoreError::InvalidResponse(format!( + Err(Error::InvalidResponse(format!( "Azure Document Intelligence analysis failed: {message}" ))) } - other => Err(CoreError::InvalidResponse(format!( + other => Err(Error::InvalidResponse(format!( "Unknown operation status: {other}" ))), } @@ -345,9 +344,9 @@ pub(super) async fn poll_document_intelligence( original_url: &str, headers: &[(String, String)], timeout: Option, -) -> CoreResult { +) -> Result { if !same_origin(operation_url, original_url) { - return Err(CoreError::InvalidResponse( + return Err(Error::InvalidResponse( "Azure Document Intelligence: rejected cross-origin polling URL".to_string(), )); } @@ -358,7 +357,7 @@ pub(super) async fn poll_document_intelligence( )); loop { if start.elapsed() > timeout { - return Err(CoreError::Network(format!( + return Err(Error::Network(format!( "Azure Document Intelligence operation polling timed out after {} seconds", timeout.as_secs() ))); @@ -373,21 +372,21 @@ pub(super) async fn poll_document_intelligence( let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let retry_after = retry_after_secs(&response); let status = response.status(); let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let response_json: Value = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}")) + Error::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}")) })?; if operation_status(&response_json)? == "succeeded" { return Ok(response_json); @@ -426,7 +425,7 @@ mod tests { assert!(matches!( error, - CoreError::InvalidRequest(message) + Error::InvalidRequest(message) if message.contains("SSRF protection") )); } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs b/litellm-rust/crates/ai-gateway/src/ocr/handler.rs index 1de34eb400e..815bc84363a 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/handler.rs @@ -1,5 +1,4 @@ -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrResponseHandling; use serde_json::Value; @@ -7,7 +6,7 @@ use super::common_utils::{poll_document_intelligence, truncate_error_body}; use super::types::ProviderOcrRequest; use crate::client::http_client; -pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> CoreResult { +pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Result { let mut request_builder = http_client().post(&request.url).json(&request.body); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -19,7 +18,7 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); if request.config.response_handling() == OcrResponseHandling::AzureDocumentIntelligencePoll @@ -31,7 +30,7 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co .and_then(|value| value.to_str().ok()) .map(str::to_string) .ok_or_else(|| { - CoreError::InvalidResponse( + Error::InvalidResponse( "Azure Document Intelligence returned 202 but no Operation-Location header found" .to_string(), ) @@ -52,17 +51,17 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let response_json: Value = serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; + .map_err(|err| Error::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; Ok(request .config diff --git a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs index 95df566dc53..401e26d3b29 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs @@ -1,11 +1,9 @@ -use std::future::Future; -use std::pin::Pin; - -use litellm_core::CoreResult; use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrAuthStrategy; use serde_json::{Map, Value, json}; +use std::future::Future; +use std::pin::Pin; use super::common_utils::{ convert_document_url_to_data_uri, has_header, ocr_provider_config, string_headers, @@ -27,7 +25,7 @@ pub(crate) struct OcrLifecycleHooks { request_metadata: RequestMetadata, } -type OcrFuture<'a, T> = Pin> + Send + 'a>>; +type OcrFuture<'a, T> = Pin> + Send + 'a>>; type OcrLogFuture<'a> = Pin + Send + 'a>>; impl OcrLifecycleHooks { @@ -46,7 +44,7 @@ impl OcrLifecycleHooks { async fn run_pre_call_guardrails( &self, request: PreparedOcrRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -74,9 +72,9 @@ impl OcrLifecycleHooks { async fn prepare_provider_request( &self, request: PreparedOcrRequest, - ) -> CoreResult { + ) -> Result { let config = ocr_provider_config(&request.custom_llm_provider, &request.model) - .ok_or_else(|| CoreError::InvalidProvider(request.custom_llm_provider.clone()))?; + .ok_or_else(|| Error::InvalidProvider(request.custom_llm_provider.clone()))?; let env_lookup = |key: &str| std::env::var(key).ok(); let headers = string_headers(request.extra_headers)?; let auth_strategy = config.auth_strategy(); @@ -120,7 +118,7 @@ impl OcrLifecycleHooks { custom_llm_provider: &str, url: &str, body: Value, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(body); } @@ -217,7 +215,7 @@ impl CallLifecycleHooks for OcrLi fn async_log_failure_event<'a>( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -278,19 +276,19 @@ fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { fn parse_ocr_pre_call_guardrail_request( request: GuardrailRequest, -) -> CoreResult<(Value, Map)> { +) -> Result<(Value, Map), Error> { let Value::Object(mut data) = request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR pre_call guardrail must return an object".to_string(), )); }; let document = data.remove("document").ok_or_else(|| { - CoreError::InvalidRequest("OCR pre_call guardrail removed document".to_string()) + Error::InvalidRequest("OCR pre_call guardrail removed document".to_string()) })?; let optional_params = match data.remove("optional_params") { Some(Value::Object(params)) => params, Some(_) => { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR pre_call guardrail optional_params must be an object".to_string(), )); } @@ -299,33 +297,32 @@ fn parse_ocr_pre_call_guardrail_request( Ok((document, optional_params)) } -fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> CoreResult { +fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> Result { let Value::Object(mut data) = request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR during_call guardrail must return an object".to_string(), )); }; - data.remove("body").ok_or_else(|| { - CoreError::InvalidRequest("OCR during_call guardrail removed body".to_string()) - }) + data.remove("body") + .ok_or_else(|| Error::InvalidRequest("OCR during_call guardrail removed body".to_string())) } -fn guardrail_error_to_core_error(error: GuardrailError) -> CoreError { - CoreError::InvalidRequest(format!("{}: {}", error.kind, error.message)) +fn guardrail_error_to_core_error(error: GuardrailError) -> Error { + Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) } -fn core_error_kind(error: &CoreError) -> &'static str { +fn core_error_kind(error: &Error) -> &'static str { match error { - CoreError::Auth(_) => "AuthError", - CoreError::InvalidProvider(_) => "InvalidProvider", - CoreError::InvalidRequest(_) => "InvalidRequest", - CoreError::InvalidType { .. } => "InvalidType", - CoreError::MissingField(_) => "MissingField", - CoreError::Http { .. } => "HttpError", - CoreError::InvalidResponse(_) => "InvalidResponse", - CoreError::Network(_) => "NetworkError", - CoreError::Connect(_) => "ConnectError", - CoreError::Routing(_) => "RoutingError", - CoreError::Unsupported(_) => "UnsupportedRequest", + Error::Auth(_) => "AuthError", + Error::InvalidProvider(_) => "InvalidProvider", + Error::InvalidRequest(_) => "InvalidRequest", + Error::InvalidType { .. } => "InvalidType", + Error::MissingField(_) => "MissingField", + Error::Http { .. } => "HttpError", + Error::InvalidResponse(_) => "InvalidResponse", + Error::Network(_) => "NetworkError", + Error::Connect(_) => "ConnectError", + Error::Routing(_) => "RoutingError", + Error::Unsupported(_) => "UnsupportedRequest", } } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs index c4c13e2300c..b59ab626fd3 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs @@ -1,4 +1,4 @@ -use litellm_core::CoreResult; +use litellm_core::Error; use litellm_core::call_lifecycle::CallLifecycle; use serde_json::Value; @@ -13,7 +13,7 @@ pub use types::OcrRequest; use handler::execute_ocr_provider_call; use prepare::{PreparedOcrCall, prepare_ocr_call}; -pub async fn ocr(request: OcrRequest<'_>) -> CoreResult { +pub async fn ocr(request: OcrRequest<'_>) -> Result { let PreparedOcrCall { request, hooks } = prepare_ocr_call(request); CallLifecycle::default() .run_request(request, &hooks, execute_ocr_provider_call) diff --git a/litellm-rust/crates/ai-gateway/src/ocr/tests.rs b/litellm-rust/crates/ai-gateway/src/ocr/tests.rs index bb2a6b06501..8c3f0425149 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/tests.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/tests.rs @@ -1,7 +1,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrResponseHandling; use serde_json::{Map, Value, json}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -395,7 +395,7 @@ async fn ocr_lifecycle_runs_failure_hook_on_provider_error() { .await .expect_err("provider error propagates"); - assert!(matches!(err, CoreError::Http { status: 500, .. })); + assert!(matches!(err, Error::Http { status: 500, .. })); server.await.expect("server task completes"); assert_eq!( logger.events(), @@ -439,7 +439,7 @@ async fn ocr_lifecycle_pre_call_block_skips_provider_socket() { .await .expect_err("guardrail blocks request"); - assert!(matches!(err, CoreError::InvalidRequest(_))); + assert!(matches!(err, Error::InvalidRequest(_))); assert_eq!(guardrail.events(), vec!["async_pre_call_hook"]); assert_eq!( logger.events(), @@ -607,7 +607,7 @@ fn string_headers_rejects_non_string_values() { let err = string_headers(Some(headers)).expect_err("non-string header rejected"); assert_eq!( err, - CoreError::InvalidRequest( + Error::InvalidRequest( "OCR extra_headers.x-retry-count must be a string, got number".to_string() ) ); diff --git a/litellm-rust/crates/ai-gateway/src/python/config.rs b/litellm-rust/crates/ai-gateway/src/python/config.rs index c028d3d6b51..d5a4dd69c8d 100644 --- a/litellm-rust/crates/ai-gateway/src/python/config.rs +++ b/litellm-rust/crates/ai-gateway/src/python/config.rs @@ -6,33 +6,31 @@ //! (and recorded in [`crate::gil`]); the realtime hot path never touches Python. //! //! Compiled only under the `python-config` feature. - -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::router::{Deployment, Router}; use pyo3::prelude::*; use crate::gil; /// Load the router's `model_list` from `config_path` via the Python reader. -pub fn load_router_from_config(config_path: &str) -> CoreResult { +pub fn load_router_from_config(config_path: &str) -> Result { gil::record_acquisition(); Python::attach(|py| { let model_list = py .import("litellm.proxy.read_model_list") .and_then(|module| module.getattr("read_model_list")) .and_then(|reader| reader.call1((config_path,))) - .map_err(|err| CoreError::Routing(format!("read_model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("read_model_list failed: {err}")))?; let model_list_json: String = py .import("json") .and_then(|json| json.getattr("dumps")) .and_then(|dumps| dumps.call1((model_list,))) .and_then(|encoded| encoded.extract()) - .map_err(|err| CoreError::Routing(format!("serializing model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("serializing model_list failed: {err}")))?; let deployments: Vec = serde_json::from_str(&model_list_json) - .map_err(|err| CoreError::Routing(format!("parsing model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("parsing model_list failed: {err}")))?; Ok(Router::new(deployments)) }) diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs index 7e38d10c6ff..e9f8c477f36 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs @@ -9,7 +9,7 @@ use axum::http::StatusCode; use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE, HeaderMap, HeaderValue}; use axum::response::{IntoResponse, Response}; use axum::routing::post; -use litellm_core::CoreError; +use litellm_core::Error; use serde_json::{Map, Value}; use crate::auth::RequireMasterKey; @@ -46,7 +46,7 @@ fn stream_response(upstream: reqwest::Response) -> Result Result Result>, CoreError> { +fn forwarded_headers(headers: &HeaderMap) -> Result>, Error> { let forwarded = headers .iter() .filter(|(name, _)| { @@ -74,19 +74,19 @@ fn forwarded_headers(headers: &HeaderMap) -> Result>, }) .map(|(name, value)| { let value = value.to_str().map_err(|_| { - CoreError::InvalidRequest(format!("invalid value for header {}", name.as_str())) + Error::InvalidRequest(format!("invalid value for header {}", name.as_str())) })?; Ok((name.to_string(), Value::String(value.to_string()))) }) - .collect::, CoreError>>()?; + .collect::, Error>>()?; Ok((!forwarded.is_empty()).then_some(forwarded)) } #[derive(Debug)] -struct MessagesRouteError(CoreError); +struct MessagesRouteError(Error); -impl From for MessagesRouteError { - fn from(error: CoreError) -> Self { +impl From for MessagesRouteError { + fn from(error: Error) -> Self { Self(error) } } @@ -94,28 +94,28 @@ impl From for MessagesRouteError { impl IntoResponse for MessagesRouteError { fn into_response(self) -> Response { let (status, message) = match self.0 { - CoreError::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message), - CoreError::InvalidProvider(_) | CoreError::Routing(_) => ( + Error::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message), + Error::InvalidProvider(_) | Error::Routing(_) => ( StatusCode::NOT_FOUND, "no messages deployment is configured for this model".to_string(), ), - CoreError::Auth(_) => ( + Error::Auth(_) => ( StatusCode::BAD_GATEWAY, "messages provider authentication failed".to_string(), ), - CoreError::Http { .. } - | CoreError::Network(_) - | CoreError::Connect(_) - | CoreError::InvalidResponse(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) => ( + Error::Http { .. } + | Error::Network(_) + | Error::Connect(_) + | Error::InvalidResponse(_) + | Error::InvalidType { .. } + | Error::MissingField(_) => ( StatusCode::BAD_GATEWAY, "messages provider request failed".to_string(), ), // The gateway has no Python implementation to decline to, so a // request the core cannot serve is reported to the caller. The // reason is a fixed internal string, never provider content. - CoreError::Unsupported(reason) => ( + Error::Unsupported(reason) => ( StatusCode::BAD_REQUEST, format!("messages request is not supported: {reason}"), ), diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs index 5f4c5fe8de4..4fd29db05d6 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs @@ -1,10 +1,10 @@ use std::sync::Arc; +use litellm_core::Error; use litellm_core::constants::ANTHROPIC_MESSAGES_PROVIDER; use litellm_core::messages::types::MessagesRequest; use litellm_core::messages::{messages, messages_stream}; use litellm_core::router::Router; -use litellm_core::{CoreError, CoreResult}; use serde_json::{Map, Value}; pub(crate) enum MessagesResponse { @@ -16,16 +16,16 @@ pub async fn run( router: &Arc, body: Value, extra_headers: Option>, -) -> CoreResult { +) -> Result { let model = body .get("model") .and_then(Value::as_str) .map(str::trim) .filter(|model| !model.is_empty()) - .ok_or_else(|| CoreError::InvalidRequest("messages body requires a model".to_string()))?; - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + .ok_or_else(|| Error::InvalidRequest("messages body requires a model".to_string()))?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let provider_model = deployment.litellm_params.model.as_str(); let upstream_model = provider_model .split_once('/') @@ -37,7 +37,7 @@ pub async fn run( }; let mut body = body; body.as_object_mut() - .ok_or_else(|| CoreError::InvalidRequest("messages body must be an object".to_string()))? + .ok_or_else(|| Error::InvalidRequest("messages body must be an object".to_string()))? .insert( "model".to_string(), Value::String(upstream_model.to_string()), @@ -60,6 +60,6 @@ pub async fn run( serde_json::to_value(response) .map(MessagesResponse::Json) .map_err(|err| { - CoreError::InvalidResponse(format!("failed to serialize messages response: {err}")) + Error::InvalidResponse(format!("failed to serialize messages response: {err}")) }) } diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs index 4ae8cfe7379..b8ee77c4269 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs @@ -11,8 +11,7 @@ use std::time::Duration; use crate::io::realtime_pool::{RealtimePool, upstream_key}; use futures_util::{Sink, Stream}; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::realtime::types::RealtimeEvent; use litellm_core::router::Router; @@ -29,15 +28,15 @@ pub async fn run( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, >::Error: std::fmt::Display, { - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let params = &deployment.litellm_params; // Strip a leading `openai/` so the OpenAI-only realtime fn gets the bare model. let provider_model = params diff --git a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs b/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs index 165c95695d3..e8f840c0c8e 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs @@ -2,13 +2,13 @@ use std::sync::Arc; use std::time::Duration; use futures_util::{Sink, Stream}; +use litellm_core::Error; use litellm_core::call_lifecycle::{CallLifecycle, CallLifecycleContext}; use litellm_core::responses::instrumentation::{ ResponsesWsCallbackPayload, ResponsesWsInstrumentation, ResponsesWsLogOutcome, ResponsesWsMetadata, }; use litellm_core::responses::types::ResponsesWsEvent; -use litellm_core::{CoreError, CoreResult}; use crate::integrations::custom_logger::{ CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, ModelCallDetails, @@ -26,22 +26,22 @@ pub async fn run( metadata: RequestMetadata, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, Out::Error: std::fmt::Display, { - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let params = &deployment.litellm_params; let provider_model = params .model .strip_prefix("openai/") .unwrap_or(¶ms.model); if params.model.contains('/') && !params.model.starts_with("openai/") { - return Err(CoreError::InvalidProvider( + return Err(Error::InvalidProvider( "Responses WebSocket route supports OpenAI deployments only".to_string(), )); } diff --git a/litellm-rust/crates/core/src/audio_transcription/transformation.rs b/litellm-rust/crates/core/src/audio_transcription/transformation.rs index eab34c13843..16a28fbcac0 100644 --- a/litellm-rust/crates/core/src/audio_transcription/transformation.rs +++ b/litellm-rust/crates/core/src/audio_transcription/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::CoreResult; - use super::types::{AudioTranscriptionRequestData, AudioTranscriptionResponseData}; #[derive(Clone, Debug, PartialEq, Eq)] @@ -32,13 +31,13 @@ pub trait AudioTranscriptionProviderConfig: Sync { model: &str, audio: Value, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_transcription_response( &self, model: &str, response_json: Value, - ) -> CoreResult; + ) -> Result; fn complete_url( &self, @@ -46,12 +45,12 @@ pub trait AudioTranscriptionProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth_strategy( &self, model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; } diff --git a/litellm-rust/crates/core/src/call_lifecycle/mod.rs b/litellm-rust/crates/core/src/call_lifecycle/mod.rs index d9b68a1b726..637c156e192 100644 --- a/litellm-rust/crates/core/src/call_lifecycle/mod.rs +++ b/litellm-rust/crates/core/src/call_lifecycle/mod.rs @@ -1,7 +1,7 @@ use std::future::Future; use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use crate::{CoreError, CoreResult}; +use crate::Error; pub mod types; @@ -11,14 +11,14 @@ pub use types::{ }; pub trait CallLifecycleHooks: Send + Sync { - type PreCallFuture<'a>: Future> + Send + 'a + type PreCallFuture<'a>: Future> + Send + 'a where Self: 'a, InitialReq: 'a, ProviderReq: 'a, Resp: 'a; - type DuringCallFuture<'a>: Future> + Send + 'a + type DuringCallFuture<'a>: Future> + Send + 'a where Self: 'a, InitialReq: 'a, @@ -56,7 +56,7 @@ pub trait CallLifecycleHooks: Send + Sync { fn async_log_failure_event<'a>( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a>; } @@ -86,12 +86,12 @@ impl<'a> CallLifecycle<'a> { request: InitialReq, hooks: &Hooks, provider_call: ProviderCall, - ) -> CoreResult + ) -> Result where InitialReq: CallLifecycleRequest, Hooks: CallLifecycleHooks, ProviderCall: FnOnce(ProviderReq) -> ProviderFuture, - ProviderFuture: Future>, + ProviderFuture: Future>, { let context = request.lifecycle_context(); self.run(context, request, hooks, provider_call).await @@ -103,11 +103,11 @@ impl<'a> CallLifecycle<'a> { request: InitialReq, hooks: &Hooks, provider_call: ProviderCall, - ) -> CoreResult + ) -> Result where Hooks: CallLifecycleHooks, ProviderCall: FnOnce(ProviderReq) -> ProviderFuture, - ProviderFuture: Future>, + ProviderFuture: Future>, { let call_start = epoch_seconds(); let mut phases = Vec::new(); @@ -166,7 +166,7 @@ impl<'a> CallLifecycle<'a> { &self, context: &CallLifecycleContext, hooks: &Hooks, - error: &CoreError, + error: &Error, call_start: f64, phases: &mut Vec, ) where @@ -251,8 +251,8 @@ mod tests { } impl CallLifecycleHooks for RecordingHooks { - type PreCallFuture<'a> = BoxFuture<'a, CoreResult>; - type DuringCallFuture<'a> = BoxFuture<'a, CoreResult>; + type PreCallFuture<'a> = BoxFuture<'a, Result>; + type DuringCallFuture<'a> = BoxFuture<'a, Result>; type SuccessFuture<'a> = BoxFuture<'a, ()>; type FailureFuture<'a> = BoxFuture<'a, ()>; @@ -294,7 +294,7 @@ mod tests { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -304,8 +304,8 @@ mod tests { } impl CallLifecycleHooks for RecordingHooks { - type PreCallFuture<'a> = BoxFuture<'a, CoreResult>; - type DuringCallFuture<'a> = BoxFuture<'a, CoreResult>; + type PreCallFuture<'a> = BoxFuture<'a, Result>; + type DuringCallFuture<'a> = BoxFuture<'a, Result>; type SuccessFuture<'a> = BoxFuture<'a, ()>; type FailureFuture<'a> = BoxFuture<'a, ()>; @@ -345,7 +345,7 @@ mod tests { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -383,13 +383,13 @@ mod tests { "request".to_string(), &hooks, |_request| async move { - Err::(CoreError::Network("provider down".to_string())) + Err::(Error::Network("provider down".to_string())) }, ) .await .expect_err("call fails"); - assert_eq!(error, CoreError::Network("provider down".to_string())); + assert_eq!(error, Error::Network("provider down".to_string())); assert_eq!(hooks.events(), vec!["pre_call", "during_call", "failure"]); } diff --git a/litellm-rust/crates/core/src/chat_completions/common_utils.rs b/litellm-rust/crates/core/src/chat_completions/common_utils.rs index 36eaf242a5a..ca51471eb7c 100644 --- a/litellm-rust/crates/core/src/chat_completions/common_utils.rs +++ b/litellm-rust/crates/core/src/chat_completions/common_utils.rs @@ -1,8 +1,7 @@ -use serde_json::{Map, Value}; - -use crate::error::CoreResult; +use crate::Error; use crate::http_utils::string_headers as shared_string_headers; use crate::providers::anthropic::chat_completions::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG; +use serde_json::{Map, Value}; use super::transformation::ChatCompletionsProviderConfig; @@ -23,6 +22,6 @@ pub(super) fn chat_completions_provider_config( pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { shared_string_headers(HEADER_CONTEXT, extra_headers) } diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index afc4529fd26..7e2731442cc 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,6 +1,6 @@ use serde_json::Value; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::http_utils::truncate_error_body; use super::client::http_client; @@ -11,9 +11,9 @@ use super::types::{ pub(super) async fn execute_chat_completions_provider_call( request: ProviderChatCompletionsRequest, -) -> CoreResult { +) -> Result { let body = serde_json::to_vec(&request.body).map_err(|err| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "failed to serialize chat completions request: {err}" )) })?; @@ -32,9 +32,9 @@ pub(super) async fn execute_chat_completions_provider_call( // so the host can still serve it. Everything else here, a timeout // above all, may have reached the provider and been answered. if err.is_connect() || err.is_builder() { - CoreError::Connect(err.to_string()) + Error::Connect(err.to_string()) } else { - CoreError::Network(err.to_string()) + Error::Network(err.to_string()) } })?; @@ -42,17 +42,17 @@ pub(super) async fn execute_chat_completions_provider_call( let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let body: Value = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid chat completions response JSON: {err}")) + Error::InvalidResponse(format!("invalid chat completions response JSON: {err}")) })?; request .config @@ -69,10 +69,10 @@ pub(super) async fn execute_chat_completions_provider_call( /// second kind has already been billed, and a host that keeps a reference /// implementation must not retry those, so collapse them to one variant that /// can only mean the provider was already called. -pub(super) fn as_response_error(err: CoreError) -> CoreError { +pub(super) fn as_response_error(err: Error) -> Error { match err { - already @ (CoreError::InvalidResponse(_) | CoreError::Http { .. }) => already, - other => CoreError::InvalidResponse(other.to_string()), + already @ (Error::InvalidResponse(_) | Error::Http { .. }) => already, + other => Error::InvalidResponse(other.to_string()), } } @@ -80,7 +80,7 @@ pub(super) fn as_response_error(err: CoreError) -> CoreError { pub(super) async fn signed_headers( request: &ProviderChatCompletionsRequest, body: &[u8], -) -> CoreResult> { +) -> Result, Error> { use std::collections::BTreeMap; use std::time::SystemTime; @@ -101,7 +101,7 @@ pub(super) async fn signed_headers( .iter() .any(|(name, _)| is_sigv4_computed_header(name)) { - return Err(CoreError::Unsupported( + return Err(Error::Unsupported( "request forwards a header AWS SigV4 computes", )); } @@ -137,9 +137,9 @@ pub(super) async fn signed_headers( pub(super) async fn signed_headers( request: &ProviderChatCompletionsRequest, _body: &[u8], -) -> CoreResult> { +) -> Result, Error> { match &request.auth { - ChatCompletionsAuth::AwsSigV4 { .. } => Err(CoreError::Unsupported( + ChatCompletionsAuth::AwsSigV4 { .. } => Err(Error::Unsupported( "AWS SigV4 requires the bedrock-auth feature", )), _ => Ok(request.upstream_headers.clone()), diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index f30ac1a24bf..0d009d36d16 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -6,6 +6,7 @@ //! credentials, and it resolves the provider, translates the conversation, //! calls the provider, and returns a typed OpenAI-shaped response. +use crate::Error; mod client; mod common_utils; pub mod conversation; @@ -17,15 +18,13 @@ pub mod types; use serde_json::{Map, Value}; -use crate::error::CoreResult; - use handler::execute_chat_completions_provider_call; use prepare::{parse_messages, prepare_chat_completions_call, resolve_provider_config}; use types::{ChatCompletionsRequest, ChatCompletionsResponse}; pub async fn chat_completions( request: ChatCompletionsRequest<'_>, -) -> CoreResult { +) -> Result { execute_chat_completions_provider_call(prepare_chat_completions_call(request)?).await } diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index 1e1c8d1bafd..142b2f2aaed 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -1,6 +1,6 @@ use serde_json::Value; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::http_utils::has_header; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; @@ -11,7 +11,7 @@ use super::types::{ChatCompletionsRequest, ChatMessage, ProviderChatCompletionsR pub(super) fn resolve_provider_config<'a>( model: &'a str, custom_llm_provider: Option<&'a str>, -) -> CoreResult<(String, &'static dyn ChatCompletionsProviderConfig)> { +) -> Result<(String, &'static dyn ChatCompletionsProviderConfig), Error> { let provider_info = get_custom_llm_provider(model, custom_llm_provider) .or_else(|| { custom_llm_provider.map(|provider| CustomLlmProvider { @@ -20,35 +20,34 @@ pub(super) fn resolve_provider_config<'a>( }) }) .ok_or_else(|| { - CoreError::InvalidProvider( + Error::InvalidProvider( "unable to resolve custom_llm_provider for chat completions request".to_string(), ) })?; let config = chat_completions_provider_config(provider_info.custom_llm_provider) - .ok_or_else(|| CoreError::InvalidProvider(provider_info.custom_llm_provider.to_string()))?; + .ok_or_else(|| Error::InvalidProvider(provider_info.custom_llm_provider.to_string()))?; Ok((provider_info.model.to_string(), config)) } -pub(super) fn parse_messages(messages: Value) -> CoreResult> { - serde_json::from_value(messages).map_err(|err| { - CoreError::InvalidRequest(format!("invalid chat completions messages: {err}")) - }) +pub(super) fn parse_messages(messages: Value) -> Result, Error> { + serde_json::from_value(messages) + .map_err(|err| Error::InvalidRequest(format!("invalid chat completions messages: {err}"))) } pub(super) fn prepare_chat_completions_call( request: ChatCompletionsRequest<'_>, -) -> CoreResult { +) -> Result { let (model, config) = resolve_provider_config(request.model, request.custom_llm_provider)?; let env_lookup = |key: &str| std::env::var(key).ok(); let messages = parse_messages(request.messages)?; if messages.is_empty() { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "chat completions requires at least one message".to_string(), )); } if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) { - return Err(CoreError::Unsupported(reason.0)); + return Err(Error::Unsupported(reason.0)); } let mut headers = string_headers(request.extra_headers)?; diff --git a/litellm-rust/crates/core/src/chat_completions/tests.rs b/litellm-rust/crates/core/src/chat_completions/tests.rs index e2383723cb0..2858d180e27 100644 --- a/litellm-rust/crates/core/src/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/chat_completions/tests.rs @@ -1,6 +1,6 @@ use serde_json::{Map, Value, json}; -use crate::error::CoreError; +use crate::error::Error; use super::prepare::prepare_chat_completions_call; use super::transformation::ChatCompletionsAuth; @@ -29,7 +29,7 @@ fn request<'a>( /// `ProviderChatCompletionsRequest` deliberately has no `Debug` (its headers /// carry resolved credentials), so unwrap the failure case by hand. -fn decline(request: ChatCompletionsRequest<'_>) -> CoreError { +fn decline(request: ChatCompletionsRequest<'_>) -> Error { match prepare_chat_completions_call(request) { Err(error) => error, Ok(prepared) => panic!("expected a decline, prepared a call to {}", prepared.url), @@ -196,7 +196,7 @@ fn declines_an_unsupported_request_before_resolving_credentials() { call.api_key = None; // No api_key is set and no env is consulted: the gate must run first, so the // error is the decline rather than a missing-credential error. - assert_eq!(decline(call), CoreError::Unsupported("streaming")); + assert_eq!(decline(call), Error::Unsupported("streaming")); } #[test] @@ -208,7 +208,7 @@ fn rejects_an_unknown_provider() { json!([{"role": "user", "content": "hi"}]), json!({}), )), - CoreError::InvalidProvider("openai".to_string()) + Error::InvalidProvider("openai".to_string()) ); } @@ -221,7 +221,7 @@ fn rejects_a_model_with_no_resolvable_provider() { json!([{"role": "user", "content": "hi"}]), json!({}), )), - CoreError::InvalidProvider(_) + Error::InvalidProvider(_) )); } @@ -234,7 +234,7 @@ fn rejects_an_empty_or_malformed_message_list() { json!([]), json!({}), )), - CoreError::InvalidRequest("chat completions requires at least one message".to_string()) + Error::InvalidRequest("chat completions requires at least one message".to_string()) ); assert!(matches!( decline(request( @@ -243,7 +243,7 @@ fn rejects_an_empty_or_malformed_message_list() { json!("not a list"), json!({}), )), - CoreError::InvalidRequest(_) + Error::InvalidRequest(_) )); } @@ -258,7 +258,7 @@ fn rejects_non_string_extra_headers() { call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))])); assert_eq!( decline(call), - CoreError::InvalidRequest( + Error::InvalidRequest( "chat completions extra_headers.x-trace must be a string, got number".to_string() ) ); @@ -374,7 +374,7 @@ async fn a_forwarded_header_the_signer_computes_declines_to_python() { .await .expect_err("{forwarded} should decline instead of being signed"); assert!( - matches!(error, CoreError::Unsupported(_)), + matches!(error, Error::Unsupported(_)), "{forwarded} declined as {error:?}, which the host would not fall back on" ); } @@ -727,7 +727,7 @@ mod round_trip { .expect_err("response cannot be normalized"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::InvalidResponse(_)), + matches!(err, Error::InvalidResponse(_)), "expected a post-send error, got {err:?}" ); } @@ -745,7 +745,7 @@ mod round_trip { .expect_err("response cannot be normalized"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::InvalidResponse(_)), + matches!(err, Error::InvalidResponse(_)), "expected a post-send error, got {err:?}" ); } @@ -763,7 +763,7 @@ mod round_trip { .expect_err("upstream rejects"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::Http { status: 429, .. }), + matches!(err, Error::Http { status: 429, .. }), "expected a 429, got {err:?}" ); } @@ -787,7 +787,7 @@ mod round_trip { .await .expect_err("nothing is listening"); assert!( - matches!(err, CoreError::Connect(_)), + matches!(err, Error::Connect(_)), "expected a pre-send connect failure, got {err:?}" ); } @@ -797,24 +797,24 @@ mod round_trip { use crate::chat_completions::handler::as_response_error; for original in [ - CoreError::MissingField("usage"), - CoreError::Unsupported("non-text response content block"), - CoreError::InvalidRequest("whatever".to_string()), - CoreError::Auth("whatever".to_string()), + Error::MissingField("usage"), + Error::Unsupported("non-text response content block"), + Error::InvalidRequest("whatever".to_string()), + Error::Auth("whatever".to_string()), ] { let label = format!("{original:?}"); assert!( - matches!(as_response_error(original), CoreError::InvalidResponse(_)), + matches!(as_response_error(original), Error::InvalidResponse(_)), "{label} must not stay retryable once the provider has answered" ); } // An upstream status is already unambiguous, so it survives intact. assert!(matches!( - as_response_error(CoreError::Http { + as_response_error(Error::Http { status: 500, body: "boom".to_string() }), - CoreError::Http { status: 500, .. } + Error::Http { status: 500, .. } )); } } diff --git a/litellm-rust/crates/core/src/chat_completions/transformation.rs b/litellm-rust/crates/core/src/chat_completions/transformation.rs index a30ce9dc77c..a0868209305 100644 --- a/litellm-rust/crates/core/src/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/chat_completions/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::error::CoreResult; - use super::types::{ ChatCompletionsResponse, ChatMessage, ChatMessageContent, ProviderChatRequestData, ProviderChatResponseData, @@ -39,7 +38,7 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth( &self, @@ -47,7 +46,7 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn default_headers(&self) -> &'static [(&'static str, &'static str)] { &[("content-type", "application/json")] @@ -91,13 +90,13 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_response( &self, model: &str, response: ProviderChatResponseData, - ) -> CoreResult; + ) -> Result; } pub fn unsupported_param( diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index 739532f8cb5..db3fa2ec704 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -1,9 +1,7 @@ -use thiserror::Error; +use thiserror::Error as ThisError; -pub type CoreResult = Result; - -#[derive(Debug, Error, PartialEq, Eq)] -pub enum CoreError { +#[derive(Debug, ThisError, PartialEq, Eq)] +pub enum Error { #[error("expected {expected}, got {actual}")] InvalidType { expected: &'static str, diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index c541f50275b..10661fadf96 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -3,7 +3,7 @@ use serde_json::{Map, Value}; use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; /// Bound an upstream error body before it crosses a host boundary, so provider /// bodies stay data-minimized. @@ -18,7 +18,7 @@ pub fn truncate_error_body(body: &str) -> String { pub fn string_headers( context: &'static str, extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { extra_headers .unwrap_or_default() .into_iter() @@ -27,7 +27,7 @@ pub fn string_headers( .as_str() .map(|value| (key.clone(), value.to_string())) .ok_or_else(|| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "{context} extra_headers.{key} must be a string, got {}", json_type_name(&value) )) @@ -81,7 +81,7 @@ mod tests { let err = string_headers("chat completions", Some(headers)).expect_err("non-string value"); assert_eq!( err, - CoreError::InvalidRequest( + Error::InvalidRequest( "chat completions extra_headers.x-trace must be a string, got number".to_string() ) ); diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index dce4a425ea0..0e18d24e5d8 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -13,4 +13,4 @@ pub mod responses; pub mod router; pub mod routing_utils; -pub use error::{CoreError, CoreResult}; +pub use error::Error; diff --git a/litellm-rust/crates/core/src/messages/common_utils.rs b/litellm-rust/crates/core/src/messages/common_utils.rs index a14dffbc1fe..8dfdb2e361a 100644 --- a/litellm-rust/crates/core/src/messages/common_utils.rs +++ b/litellm-rust/crates/core/src/messages/common_utils.rs @@ -1,9 +1,8 @@ -use serde_json::{Map, Value}; - -use crate::error::CoreResult; +use crate::Error; use crate::http_utils::string_headers as shared_string_headers; use crate::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG; use crate::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG; +use serde_json::{Map, Value}; use super::transformation::AnthropicMessagesProviderConfig; @@ -23,6 +22,6 @@ pub(super) fn messages_provider_config( pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { shared_string_headers(HEADER_CONTEXT, extra_headers) } diff --git a/litellm-rust/crates/core/src/messages/handler.rs b/litellm-rust/crates/core/src/messages/handler.rs index 1c895f66eba..13a65d86131 100644 --- a/litellm-rust/crates/core/src/messages/handler.rs +++ b/litellm-rust/crates/core/src/messages/handler.rs @@ -1,5 +1,5 @@ use crate::constants::ANTHROPIC_MESSAGES_PROVIDER; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use super::client::http_client; use super::common_utils::truncate_error_body; @@ -7,7 +7,7 @@ use super::types::{AnthropicMessagesResponse, ProviderMessagesRequest}; pub(super) async fn execute_messages_provider_call( request: ProviderMessagesRequest, -) -> CoreResult { +) -> Result { let mut request_builder = http_client().post(&request.url).json(&request.body); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -19,32 +19,31 @@ pub(super) async fn execute_messages_provider_call( let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } - let response = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid messages response JSON: {err}")) - })?; + let response = serde_json::from_str(&text) + .map_err(|err| Error::InvalidResponse(format!("invalid messages response JSON: {err}")))?; request.config.transform_response(&request.model, response) } pub(super) async fn execute_messages_provider_stream( request: ProviderMessagesRequest, -) -> CoreResult { +) -> Result { if request.provider != ANTHROPIC_MESSAGES_PROVIDER { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "streaming messages is not supported for this provider".to_string(), )); } @@ -60,14 +59,14 @@ pub(super) async fn execute_messages_provider_stream( let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); if !status.is_success() { let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; - return Err(CoreError::Http { + .map_err(|err| Error::Network(err.to_string()))?; + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index acb36d89daf..ee2877e61fc 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -7,6 +7,7 @@ //! is the streaming variant; it hands the raw upstream response back so a host //! can splice the event stream to its own caller. +use crate::Error; mod client; mod common_utils; mod handler; @@ -14,17 +15,15 @@ mod prepare; pub mod transformation; pub mod types; -use crate::error::CoreResult; - use handler::{execute_messages_provider_call, execute_messages_provider_stream}; use prepare::prepare_messages_call; use types::{AnthropicMessagesResponse, MessagesRequest}; -pub async fn messages(request: MessagesRequest<'_>) -> CoreResult { +pub async fn messages(request: MessagesRequest<'_>) -> Result { execute_messages_provider_call(prepare_messages_call(request)?).await } -pub async fn messages_stream(request: MessagesRequest<'_>) -> CoreResult { +pub async fn messages_stream(request: MessagesRequest<'_>) -> Result { execute_messages_provider_stream(prepare_messages_call(request)?).await } diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index 94b5b1eaed7..3b253ac3766 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}; @@ -7,7 +7,7 @@ use super::types::{MessagesRequest, ProviderMessagesRequest}; pub(super) fn prepare_messages_call( request: MessagesRequest<'_>, -) -> CoreResult { +) -> Result { let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider) .or_else(|| { request @@ -18,7 +18,7 @@ pub(super) fn prepare_messages_call( }) }) .ok_or_else(|| { - CoreError::InvalidProvider( + Error::InvalidProvider( "unable to resolve custom_llm_provider for messages request".to_string(), ) })?; @@ -26,7 +26,7 @@ pub(super) fn prepare_messages_call( let provider = provider_info.custom_llm_provider; let config = messages_provider_config(provider) - .ok_or_else(|| CoreError::InvalidProvider(provider.to_string()))?; + .ok_or_else(|| Error::InvalidProvider(provider.to_string()))?; let env_lookup = |key: &str| std::env::var(key).ok(); let mut headers = string_headers(request.extra_headers)?; @@ -53,11 +53,11 @@ pub(super) fn prepare_messages_call( let url = config.complete_url(request.api_base, &model, &env_lookup)?; let typed_request = serde_json::from_value(request.body).map_err(|err| { - CoreError::InvalidRequest(format!("invalid Anthropic messages request: {err}")) + Error::InvalidRequest(format!("invalid Anthropic messages request: {err}")) })?; let transformed = config.transform_request(typed_request)?; let body = serde_json::to_value(transformed).map_err(|err| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "failed to serialize Anthropic messages request: {err}" )) })?; diff --git a/litellm-rust/crates/core/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs index 9fc1763683b..df9f7051011 100644 --- a/litellm-rust/crates/core/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -4,7 +4,7 @@ use serde_json::{Map, Value, json}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; -use crate::error::CoreError; +use crate::error::Error; use super::common_utils::{ has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body, @@ -77,7 +77,7 @@ fn truncate_error_body_caps_long_payloads() { fn string_headers_rejects_non_string_values() { let headers = json!({"x-count": 3}).as_object().unwrap().clone(); let err = string_headers(Some(headers)).expect_err("non-string header rejected"); - assert!(matches!(err, CoreError::InvalidRequest(_))); + assert!(matches!(err, Error::InvalidRequest(_))); } #[test] @@ -341,7 +341,7 @@ async fn messages_requires_auth_when_no_key_and_no_header() { .await .expect_err("missing auth errors"); - assert!(matches!(err, CoreError::Auth(_))); + assert!(matches!(err, Error::Auth(_))); } #[tokio::test] @@ -420,7 +420,7 @@ async fn messages_maps_provider_error_status_to_http_error() { .await .expect_err("provider error propagates"); - assert!(matches!(err, CoreError::Http { status: 401, .. })); + assert!(matches!(err, Error::Http { status: 401, .. })); } #[tokio::test] @@ -437,5 +437,5 @@ async fn messages_rejects_unsupported_provider() { .await .expect_err("unsupported provider errors"); - assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "openai")); + assert!(matches!(err, Error::InvalidProvider(provider) if provider == "openai")); } diff --git a/litellm-rust/crates/core/src/messages/transformation.rs b/litellm-rust/crates/core/src/messages/transformation.rs index b478e20d24b..673a5728aca 100644 --- a/litellm-rust/crates/core/src/messages/transformation.rs +++ b/litellm-rust/crates/core/src/messages/transformation.rs @@ -1,6 +1,5 @@ -use crate::error::CoreResult; - use super::types::{AnthropicMessagesRequest, AnthropicMessagesResponse}; +use crate::Error; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MessagesAuthStrategy { @@ -23,13 +22,13 @@ pub trait AnthropicMessagesProviderConfig: Sync { api_base: Option<&str>, model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn resolve_api_key( &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth_strategy(&self) -> MessagesAuthStrategy { MessagesAuthStrategy::Header("x-api-key") @@ -49,7 +48,7 @@ pub trait AnthropicMessagesProviderConfig: Sync { fn transform_request( &self, request: AnthropicMessagesRequest, - ) -> CoreResult { + ) -> Result { Ok(request) } @@ -57,7 +56,7 @@ pub trait AnthropicMessagesProviderConfig: Sync { &self, _model: &str, response: AnthropicMessagesResponse, - ) -> CoreResult { + ) -> Result { Ok(response) } } diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs index cb3e735e533..3d3c16c8cb6 100644 --- a/litellm-rust/crates/core/src/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::CoreResult; - use super::types::{OcrRequestData, OcrResponseData}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -43,13 +42,13 @@ pub trait OcrProviderConfig: Sync { model: &str, document: Value, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_ocr_response( &self, model: &str, response_json: Value, - ) -> CoreResult; + ) -> Result; fn complete_url( &self, @@ -57,13 +56,13 @@ pub trait OcrProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn resolve_api_key( &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth_strategy(&self) -> OcrAuthStrategy { OcrAuthStrategy::Bearer diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs index 4534ac0182c..b22de6c47de 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::Error; use serde_json::json; fn messages(value: Value) -> Vec { @@ -19,7 +20,7 @@ fn transform(model: &str, msgs: Value, opts: Value) -> Value { .body } -fn transform_response(body: Value) -> CoreResult { +fn transform_response(body: Value) -> Result { ANTHROPIC_CHAT_COMPLETIONS_CONFIG .transform_response("claude-sonnet-4-5", ProviderChatResponseData { body }) } @@ -390,29 +391,26 @@ fn declines_a_response_carrying_a_non_text_block() { "usage": {"input_tokens": 1, "output_tokens": 1} })) .expect_err("non-text block"); - assert_eq!( - err, - CoreError::Unsupported("non-text response content block") - ); + assert_eq!(err, Error::Unsupported("non-text response content block")); } #[test] fn errors_on_a_response_missing_required_fields() { assert_eq!( transform_response(json!("nope")).expect_err("not an object"), - CoreError::InvalidResponse("messages response is not an object".to_string()) + Error::InvalidResponse("messages response is not an object".to_string()) ); assert_eq!( transform_response(json!({"model": "m", "usage": {}})).expect_err("no content"), - CoreError::MissingField("content") + Error::MissingField("content") ); assert_eq!( transform_response(json!({"model": "m", "content": []})).expect_err("no usage"), - CoreError::MissingField("usage") + Error::MissingField("usage") ); assert_eq!( transform_response(json!({"content": [], "usage": {}})).expect_err("no model"), - CoreError::MissingField("model") + Error::MissingField("model") ); } diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs index 3658642b539..97cc48aa6f2 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs @@ -10,7 +10,7 @@ use crate::chat_completions::types::{ ProviderChatRequestData, ProviderChatResponseData, }; use crate::constants::ANTHROPIC_OAUTH_TOKEN_PREFIX; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::providers::anthropic::messages::transformation::{ complete_anthropic_url, resolve_anthropic_api_key, }; @@ -74,7 +74,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_anthropic_url(api_base, env_lookup)) } @@ -84,7 +84,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(ChatCompletionsAuth::Header { name: "x-api-key", value: resolve_anthropic_api_key(api_key, env_lookup)?, @@ -137,7 +137,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult { + ) -> Result { Ok(ProviderChatRequestData { body: anthropic_body(model, &build_conversation(&messages), optional_params), }) @@ -147,15 +147,16 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { &self, _model: &str, response: ProviderChatResponseData, - ) -> CoreResult { - let body = response.body.as_object().ok_or_else(|| { - CoreError::InvalidResponse("messages response is not an object".into()) - })?; + ) -> Result { + let body = response + .body + .as_object() + .ok_or_else(|| Error::InvalidResponse("messages response is not an object".into()))?; let content = body .get("content") .and_then(Value::as_array) - .ok_or(CoreError::MissingField("content"))?; + .ok_or(Error::MissingField("content"))?; // The route declines tool and thinking requests, so a non-text block // means the response carries something this path never asked for. // Decline rather than silently dropping it; the host falls back. @@ -163,7 +164,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { .iter() .any(|block| block.get("type").and_then(Value::as_str) != Some("text")) { - return Err(CoreError::Unsupported("non-text response content block")); + return Err(Error::Unsupported("non-text response content block")); } let text: String = content .iter() @@ -173,7 +174,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { let usage = body .get("usage") .and_then(Value::as_object) - .ok_or(CoreError::MissingField("usage"))?; + .ok_or(Error::MissingField("usage"))?; let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0); Ok(ChatCompletionsResponse { @@ -181,7 +182,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { model: body .get("model") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("model"))? + .ok_or(Error::MissingField("model"))? .to_string(), choices: vec![ChatCompletionsChoice { index: 0, diff --git a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs index 829f2260d3c..8fcc0f36c7d 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; @@ -17,12 +17,12 @@ pub fn non_empty(value: Option<&str>) -> Option<&str> { pub fn resolve_anthropic_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(ANTHROPIC_API_KEY_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY \ environment variable" .to_string(), @@ -52,7 +52,7 @@ impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { api_base: Option<&str>, _model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_anthropic_url(api_base, env_lookup)) } @@ -60,7 +60,7 @@ impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_anthropic_api_key(api_key, env_lookup) } @@ -121,7 +121,7 @@ mod tests { ); assert!(matches!( resolve_anthropic_api_key(None, &|_| None).expect_err("missing key"), - CoreError::Auth(_) + Error::Auth(_) )); } diff --git a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs index 7b958c77ba3..70dad0300f1 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; use crate::messages::types::{ AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock, @@ -28,12 +28,12 @@ pub const AZURE_ANTHROPIC_MESSAGES_CONFIG: AzureAnthropicMessagesConfig = pub fn resolve_azure_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_KEY_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Azure API Key - Set `api_key` or the AZURE_API_KEY environment variable" .to_string(), ) @@ -43,12 +43,12 @@ pub fn resolve_azure_api_key( pub fn complete_azure_anthropic_url( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let api_base = non_empty(api_base) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_BASE_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Azure API Base - Set `api_base` or the AZURE_API_BASE environment variable. \ Expected format: https://.services.ai.azure.com/anthropic" .to_string(), @@ -147,7 +147,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { api_base: Option<&str>, _model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_azure_anthropic_url(api_base, env_lookup) } @@ -155,7 +155,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_azure_api_key(api_key, env_lookup) } @@ -174,7 +174,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { fn transform_request( &self, request: AnthropicMessagesRequest, - ) -> CoreResult { + ) -> Result { let mut request = fold_system_role_messages(request); if let Some(system) = request.system.as_mut() { strip_scope_from_system(system); @@ -190,7 +190,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { &self, model: &str, response: AnthropicMessagesResponse, - ) -> CoreResult { + ) -> Result { self.anthropic.transform_response(model, response) } } @@ -268,7 +268,7 @@ mod tests { "https://env.services.ai.azure.com/anthropic/v1/messages" ); let err = complete_azure_anthropic_url(Some(" "), &|_| None).expect_err("missing base"); - assert!(matches!(err, CoreError::Auth(_))); + assert!(matches!(err, Error::Auth(_))); } #[test] @@ -284,7 +284,7 @@ mod tests { ); assert!(matches!( resolve_azure_api_key(None, &|_| None).expect_err("missing key"), - CoreError::Auth(_) + Error::Auth(_) )); } diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs index eabd15677cc..b26a7925e8a 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs @@ -1,6 +1,6 @@ use std::collections::BTreeSet; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::{OcrAuthStrategy, OcrProviderConfig, OcrResponseHandling}; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value, json}; @@ -32,17 +32,17 @@ fn resolve_value( env_name: &str, env_lookup: &dyn Fn(&str) -> Option, missing_message: &str, -) -> CoreResult { +) -> Result { non_empty(explicit) .map(str::to_string) .or_else(|| env_lookup(env_name).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| CoreError::Auth(missing_message.to_string())) + .ok_or_else(|| Error::Auth(missing_message.to_string())) } pub fn resolve_azure_ai_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_key, AZURE_AI_API_KEY_ENV, @@ -54,7 +54,7 @@ pub fn resolve_azure_ai_api_key( pub fn resolve_azure_ai_api_base( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_base, AZURE_AI_API_BASE_ENV, @@ -66,7 +66,7 @@ pub fn resolve_azure_ai_api_base( pub fn complete_azure_ai_url( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let base = resolve_azure_ai_api_base(api_base, env_lookup)?; Ok(format!( "{}/providers/mistral/azure/ocr", @@ -77,7 +77,7 @@ pub fn complete_azure_ai_url( pub fn resolve_document_intelligence_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_key, AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV, @@ -89,7 +89,7 @@ pub fn resolve_document_intelligence_api_key( pub fn resolve_document_intelligence_endpoint( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_base, AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV, @@ -127,7 +127,7 @@ fn pages_token_is_valid(token: &str) -> bool { } } -fn normalize_pages_param(pages: &Value) -> CoreResult> { +fn normalize_pages_param(pages: &Value) -> Result, Error> { match pages { Value::String(value) => { let normalized = value @@ -138,7 +138,7 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { if normalized.split(',').all(pages_token_is_valid) { Ok(Some(normalized)) } else { - Err(CoreError::InvalidRequest(format!( + Err(Error::InvalidRequest(format!( "Invalid `pages` string for Azure Document Intelligence: {value:?}. Expected format like '1-3,5,7-9'." ))) } @@ -152,7 +152,7 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { for value in values { let page = value.as_i64().expect("checked is_i64"); if page < 0 { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "`pages` integers must be >= 0 (Mistral 0-based indices)".to_string(), )); } @@ -176,16 +176,16 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { if normalized.split(',').all(pages_token_is_valid) { return Ok(Some(normalized)); } - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Invalid `pages` list for Azure Document Intelligence: {values:?}. Expected tokens like '1' or '3-5'." ))); } - Err(CoreError::InvalidRequest( + Err(Error::InvalidRequest( "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." .to_string(), )) } - _ => Err(CoreError::InvalidRequest( + _ => Err(Error::InvalidRequest( "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." .to_string(), )), @@ -197,7 +197,7 @@ pub fn complete_document_intelligence_url( model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let endpoint = resolve_document_intelligence_endpoint(api_base, env_lookup)?; let mut url = format!( "{}/documentintelligence/documentModels/{}:analyze?api-version={}", @@ -216,20 +216,20 @@ pub fn complete_document_intelligence_url( Ok(url) } -fn document_url_from_mistral_document(document: &Value) -> CoreResult<&str> { - let object = document.as_object().ok_or_else(|| CoreError::InvalidType { +fn document_url_from_mistral_document(document: &Value) -> Result<&str, Error> { + let object = document.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(document), })?; let doc_type = object .get("type") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("document.type"))?; + .ok_or(Error::MissingField("document.type"))?; let field_name = match doc_type { "document_url" => "document_url", "image_url" => "image_url", other => { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Invalid document type: {other}. Must be 'document_url' or 'image_url'" ))); } @@ -238,7 +238,7 @@ fn document_url_from_mistral_document(document: &Value) -> CoreResult<&str> { .get(field_name) .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField(field_name)) + .ok_or(Error::MissingField(field_name)) } fn extract_base64_from_data_uri(data_uri: &str) -> &str { @@ -290,7 +290,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } @@ -298,7 +298,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -308,7 +308,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_azure_ai_url(api_base, env_lookup) } @@ -316,7 +316,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_azure_ai_api_key(api_key, env_lookup) } @@ -335,7 +335,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { _model: &str, document: Value, _optional_params: Map, - ) -> CoreResult { + ) -> Result { let document_url = document_url_from_mistral_document(&document)?; let mut data = Map::new(); if document_url.starts_with("data:") { @@ -359,19 +359,19 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; let status = response .get("status") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("status"))?; + .ok_or(Error::MissingField("status"))?; if status != "succeeded" { - return Err(CoreError::InvalidResponse(format!( + return Err(Error::InvalidResponse(format!( "Azure Document Intelligence analysis failed with status: {status}" ))); } @@ -414,7 +414,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_document_intelligence_url(api_base, model, optional_params, env_lookup) } @@ -422,7 +422,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_document_intelligence_api_key(api_key, env_lookup) } diff --git a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs index 5e885734182..bb4f6afe5f9 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs @@ -6,7 +6,7 @@ use crate::audio_transcription::transformation::{ use crate::audio_transcription::types::{ AudioTranscriptionRequestData, AudioTranscriptionResponseData, }; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; pub use super::aws_base::{aws_auth_config, bedrock_model_id_and_region, resolve_bedrock_region}; use super::constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}; @@ -18,8 +18,8 @@ pub static BEDROCK_AUDIO_TRANSCRIPTION_CONFIG: BedrockAudioTranscriptionConfig = pub struct BedrockAudioTranscriptionConfig; -fn audio_fields(audio: Value) -> CoreResult<(String, String)> { - let object = audio.as_object().ok_or_else(|| CoreError::InvalidType { +fn audio_fields(audio: Value) -> Result<(String, String), Error> { + let object = audio.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&audio), })?; @@ -27,13 +27,13 @@ fn audio_fields(audio: Value) -> CoreResult<(String, String)> { .get("data") .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField("audio.data"))?; + .ok_or(Error::MissingField("audio.data"))?; let format = object .get("format") .and_then(Value::as_str) .filter(|value| matches!(*value, "wav" | "mp3" | "flac" | "ogg")) .ok_or_else(|| { - CoreError::InvalidRequest("audio.format must be wav, mp3, flac, or ogg".to_string()) + Error::InvalidRequest("audio.format must be wav, mp3, flac, or ogg".to_string()) })?; Ok((data.to_string(), format.to_string())) } @@ -55,7 +55,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { _model: &str, audio: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { let (data, format) = audio_fields(audio)?; let mut instruction = "Transcribe the audio. Respond with only the transcript.".to_string(); if let Some(language) = optional_string(&optional_params, "language") { @@ -87,14 +87,14 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { &self, _model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let content = response_json .get("output") .and_then(|value| value.get("message")) .and_then(|value| value.get("content")) .and_then(Value::as_array) .ok_or_else(|| { - CoreError::InvalidResponse("Bedrock response has no output content".to_string()) + Error::InvalidResponse("Bedrock response has no output content".to_string()) })?; let mut text = String::new(); for block in content { @@ -111,7 +111,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (model_id, model_region) = bedrock_model_id_and_region(model); let region = resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup); let endpoint = optional_params @@ -133,7 +133,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (_, model_region) = bedrock_model_id_and_region(model); Ok(AudioTranscriptionAuth::AwsSigV4 { region: resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup), diff --git a/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs b/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs index b11639aa09b..e5e52bfce95 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs @@ -4,7 +4,7 @@ use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use crate::caching::in_memory_cache::InMemoryCache; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use aws_credential_types::Credentials; use aws_credential_types::provider::ProvideCredentials; use aws_sigv4::http_request::{ @@ -197,7 +197,7 @@ pub fn classify_auth( pub async fn resolve_credentials( config: AwsAuthConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), -) -> CoreResult { +) -> Result { let resolved = config.clone().with_environment(env_lookup); let flow = classify_auth(config, env_lookup); match flow { @@ -244,9 +244,10 @@ pub async fn resolve_credentials( let provider = aws_config::profile::ProfileFileCredentialsProvider::builder() .profile_name(name) .build(); - provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS profile credentials failed: {error}")) - }) + provider + .provide_credentials() + .await + .map_err(|error| Error::Auth(format!("AWS profile credentials failed: {error}"))) } AwsAuthFlow::AssumeRole { role, session_name } => { if is_already_running_as_role(&role, &resolved).await? { @@ -260,7 +261,7 @@ pub async fn resolve_credentials( .build() .await; let credentials = provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS default credentials failed: {error}")) + Error::Auth(format!("AWS default credentials failed: {error}")) })?; set_cached_credentials( key, @@ -301,7 +302,7 @@ pub async fn resolve_credentials( provider .provide_credentials() .await - .map_err(|error| CoreError::Auth(format!("AWS role credentials failed: {error}"))) + .map_err(|error| Error::Auth(format!("AWS role credentials failed: {error}"))) } AwsAuthFlow::WebIdentity { token, @@ -325,13 +326,13 @@ pub async fn resolve_credentials( .send() .await .map_err(|error| { - CoreError::Auth(format!("AWS web identity credentials failed: {error}")) + Error::Auth(format!("AWS web identity credentials failed: {error}")) })?; let credentials = response.credentials().ok_or_else(|| { - CoreError::Auth("AWS web identity response had no credentials".to_string()) + Error::Auth("AWS web identity response had no credentials".to_string()) })?; let expiration = SystemTime::try_from(*credentials.expiration()).map_err(|error| { - CoreError::Auth(format!("AWS web identity expiration was invalid: {error}")) + Error::Auth(format!("AWS web identity expiration was invalid: {error}")) })?; Ok(Credentials::new( credentials.access_key_id(), @@ -350,9 +351,10 @@ pub async fn resolve_credentials( aws_config::default_provider::credentials::DefaultCredentialsChain::builder() .build() .await; - let credentials = provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS default credentials failed: {error}")) - })?; + let credentials = provider + .provide_credentials() + .await + .map_err(|error| Error::Auth(format!("AWS default credentials failed: {error}")))?; set_cached_credentials( key, credentials.clone(), @@ -363,7 +365,7 @@ pub async fn resolve_credentials( } } -async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> CoreResult { +async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> Result { if role_identity(role).is_none() { return Ok(false); } @@ -437,7 +439,7 @@ pub fn sign_bedrock_post( region: &str, credentials: &Credentials, signing_time: SystemTime, -) -> CoreResult> { +) -> Result, Error> { let identity: Identity = credentials.clone().into(); let params = v4::SigningParams::builder() .identity(&identity) @@ -447,14 +449,14 @@ pub fn sign_bedrock_post( .settings(SigningSettings::default()) .build() .map(SigningParams::from) - .map_err(|error| CoreError::Auth(format!("AWS signing parameters failed: {error}")))?; + .map_err(|error| Error::Auth(format!("AWS signing parameters failed: {error}")))?; let header_refs = headers .iter() .map(|(name, value)| (name.as_str(), value.as_str())); let request = SignableRequest::new("POST", url, header_refs, SignableBody::Bytes(body)) - .map_err(|error| CoreError::Auth(format!("AWS signable request failed: {error}")))?; + .map_err(|error| Error::Auth(format!("AWS signable request failed: {error}")))?; let (instructions, _) = sign(request, ¶ms) - .map_err(|error| CoreError::Auth(format!("AWS request signing failed: {error}")))? + .map_err(|error| Error::Auth(format!("AWS request signing failed: {error}")))? .into_parts(); Ok(instructions .headers() diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs index 4b75dcb8e9d..c86f061b9ca 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::Error; use serde_json::json; fn messages(value: Value) -> Vec { @@ -23,7 +24,7 @@ fn transform(msgs: Value, opts: Value) -> Value { .body } -fn transform_response(body: Value) -> CoreResult { +fn transform_response(body: Value) -> Result { BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response( "anthropic.claude-sonnet-4-5-v1:0", ProviderChatResponseData { body }, @@ -478,25 +479,22 @@ fn declines_a_response_carrying_a_tool_use_block() { "usage": {"inputTokens": 1, "outputTokens": 1} })) .expect_err("tool use block"); - assert_eq!( - err, - CoreError::Unsupported("non-text response content block") - ); + assert_eq!(err, Error::Unsupported("non-text response content block")); } #[test] fn errors_on_a_response_missing_required_fields() { assert_eq!( transform_response(json!("nope")).expect_err("not an object"), - CoreError::InvalidResponse("converse response is not an object".to_string()) + Error::InvalidResponse("converse response is not an object".to_string()) ); assert_eq!( transform_response(json!({"usage": {}})).expect_err("no output"), - CoreError::MissingField("output.message.content") + Error::MissingField("output.message.content") ); assert_eq!( transform_response(json!({"output": {"message": {"content": []}}})).expect_err("no usage"), - CoreError::MissingField("usage") + Error::MissingField("usage") ); } diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs index b107950748e..ef5f44b4a14 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs @@ -11,7 +11,7 @@ use crate::chat_completions::types::{ ChatCompletionsUsage, ChatMessage, ChatMessageContent, ProviderChatRequestData, ProviderChatResponseData, }; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use super::super::aws_base::{bedrock_model_id_and_region, resolve_bedrock_region}; use super::super::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE}; @@ -110,7 +110,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (model_id, model_region) = bedrock_model_id_and_region(model); let region = resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup); let endpoint = optional_params @@ -137,7 +137,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { // Python reads `api_key` as the Bedrock bearer token and consults the // env only when the caller passed none, so a caller-supplied empty key // falls through to SigV4 without reaching for the environment. An @@ -208,7 +208,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { _model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult { + ) -> Result { Ok(ProviderChatRequestData { body: converse_body(&build_conversation(&messages), &optional_params), }) @@ -218,17 +218,18 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { &self, model: &str, response: ProviderChatResponseData, - ) -> CoreResult { - let body = response.body.as_object().ok_or_else(|| { - CoreError::InvalidResponse("converse response is not an object".into()) - })?; + ) -> Result { + let body = response + .body + .as_object() + .ok_or_else(|| Error::InvalidResponse("converse response is not an object".into()))?; let content = body .get("output") .and_then(|output| output.get("message")) .and_then(|message| message.get("content")) .and_then(Value::as_array) - .ok_or(CoreError::MissingField("output.message.content"))?; + .ok_or(Error::MissingField("output.message.content"))?; // The route declines tool requests, so anything other than a text block // is something this path never asked for. Decline; the host falls back. if content.iter().any(|block| { @@ -236,7 +237,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { .as_object() .is_none_or(|block| block.len() != 1 || !block.contains_key("text")) }) { - return Err(CoreError::Unsupported("non-text response content block")); + return Err(Error::Unsupported("non-text response content block")); } let text: String = content .iter() @@ -246,7 +247,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { let usage = body .get("usage") .and_then(Value::as_object) - .ok_or(CoreError::MissingField("usage"))?; + .ok_or(Error::MissingField("usage"))?; let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0); let computed = usage_from_parts( field("inputTokens"), diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs index dc720cc4244..6a8a38204a9 100644 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value}; @@ -47,7 +47,7 @@ pub fn complete_url(api_base: Option<&str>) -> String { /// Resolve the Mistral API key from the explicit param or the environment. /// -/// Blank/whitespace values are treated as absent. Returns `CoreError::Auth` +/// Blank/whitespace values are treated as absent. Returns `Error::Auth` /// when no usable key is available. /// /// Note: the env fallback only reads the process environment. Secret-manager @@ -56,13 +56,13 @@ pub fn complete_url(api_base: Option<&str>) -> String { pub fn resolve_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) .map(str::to_string) .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty())) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } pub struct MistralOcrConfig; @@ -79,9 +79,9 @@ impl OcrProviderConfig for MistralOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { if !document.is_object() { - return Err(CoreError::InvalidType { + return Err(Error::InvalidType { expected: "object", actual: json_type_name(&document), }); @@ -104,10 +104,10 @@ impl OcrProviderConfig for MistralOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response_object = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; @@ -140,7 +140,7 @@ impl OcrProviderConfig for MistralOcrConfig { _model: &str, _optional_params: &Map, _env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_url(api_base)) } @@ -148,7 +148,7 @@ impl OcrProviderConfig for MistralOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_api_key(api_key, env_lookup) } } @@ -165,11 +165,11 @@ pub fn transform_ocr_request( model: &str, document: Value, optional_params: Map, -) -> CoreResult { +) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } -pub fn transform_ocr_response(model: &str, response_json: Value) -> CoreResult { +pub fn transform_ocr_response(model: &str, response_json: Value) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -250,7 +250,7 @@ mod tests { assert_eq!( err, - CoreError::InvalidType { + Error::InvalidType { expected: "object", actual: "string", } @@ -307,6 +307,6 @@ mod tests { #[test] fn resolve_api_key_errors_when_absent() { let err = resolve_api_key(None, &|_| None).expect_err("missing key should error"); - assert_eq!(err, CoreError::Auth(MISSING_KEY_MESSAGE.to_string())); + assert_eq!(err, Error::Auth(MISSING_KEY_MESSAGE.to_string())); } } diff --git a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs index b3f6b03b28a..f1985f81b7d 100644 --- a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs +++ b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::realtime::transformation::RealtimeProviderConfig; use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; @@ -72,7 +72,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { &self, event: &RealtimeEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(RealtimeTransformResult::passthrough(event.clone())) } @@ -80,7 +80,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { &self, event: &RealtimeEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(RealtimeTransformResult::passthrough(event.clone())) } } @@ -88,14 +88,14 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { pub fn transform_realtime_request( event: &RealtimeEvent, model: &str, -) -> CoreResult { +) -> Result { OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model) } pub fn transform_realtime_response( event: &RealtimeEvent, model: &str, -) -> CoreResult { +) -> Result { OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model) } diff --git a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs index e15197c468c..be86bb90311 100644 --- a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs +++ b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::responses::types::{ResponsesWsEvent, ResponsesWsTransformResult}; use crate::responses::websocket::{ResponsesWebSocketProviderConfig, enforce_model}; @@ -15,7 +15,7 @@ impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig { &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult { + ) -> Result { Ok(ResponsesWsTransformResult::passthrough(enforce_model( event, model, ))) @@ -25,7 +25,7 @@ impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig { &self, event: &ResponsesWsEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(ResponsesWsTransformResult::passthrough(event.clone())) } } diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs index 6300149c237..ee095447028 100644 --- a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value, json}; @@ -43,7 +43,7 @@ pub fn is_deepseek_model(model: &str) -> bool { pub fn resolve_vertex_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) @@ -51,7 +51,7 @@ pub fn resolve_vertex_api_key( .or_else(|| env_lookup(VERTEX_AI_API_KEY_ENV).filter(|key| !key.trim().is_empty())) .or_else(|| env_lookup(VERTEXAI_API_KEY_ENV).filter(|key| !key.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Vertex AI access token - pass api_key or provide Authorization via extra_headers" .to_string(), ) @@ -61,12 +61,12 @@ pub fn resolve_vertex_api_key( fn vertex_project( params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { string_param(params, &["vertex_project", "vertex_ai_project"]) .map(str::to_string) .or_else(|| env_lookup(VERTEXAI_PROJECT_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::InvalidRequest( + Error::InvalidRequest( "Missing vertex_project - Set VERTEXAI_PROJECT environment variable or pass vertex_project parameter" .to_string(), ) @@ -99,7 +99,7 @@ pub fn complete_vertex_mistral_url( model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let project = vertex_project(optional_params, env_lookup)?; let location = vertex_location(optional_params, env_lookup); let base = vertex_mistral_api_base(api_base, &location); @@ -112,7 +112,7 @@ pub fn complete_vertex_deepseek_url( api_base: Option<&str>, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let project = vertex_project(optional_params, env_lookup)?; let location = vertex_location(optional_params, env_lookup); let base = api_base @@ -125,20 +125,20 @@ pub fn complete_vertex_deepseek_url( )) } -fn document_content_item(document: &Value) -> CoreResult { - let object = document.as_object().ok_or_else(|| CoreError::InvalidType { +fn document_content_item(document: &Value) -> Result { + let object = document.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(document), })?; let doc_type = object .get("type") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("document.type"))?; + .ok_or(Error::MissingField("document.type"))?; let url_field = match doc_type { "image_url" => "image_url", "document_url" => "document_url", other => { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Unsupported document type: {other}. Expected 'image_url' or 'document_url'" ))); } @@ -147,7 +147,7 @@ fn document_content_item(document: &Value) -> CoreResult { .get(url_field) .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField(url_field))?; + .ok_or(Error::MissingField(url_field))?; Ok(json!({ "type": "image_url", @@ -163,7 +163,7 @@ fn deepseek_model_name(model: &str) -> String { } } -fn first_choice_content(response: &Value) -> CoreResult { +fn first_choice_content(response: &Value) -> Result { response .get("choices") .and_then(Value::as_array) @@ -176,9 +176,7 @@ fn first_choice_content(response: &Value) -> CoreResult { Value::Object(_) => true, _ => false, }) - .ok_or_else(|| { - CoreError::InvalidResponse("No content in DeepSeek OCR response".to_string()) - }) + .ok_or_else(|| Error::InvalidResponse("No content in DeepSeek OCR response".to_string())) } fn ocr_data_from_content(content: Value, usage: Option, model: &str) -> Value { @@ -219,7 +217,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } @@ -227,7 +225,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -237,7 +235,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_vertex_mistral_url(api_base, model, optional_params, env_lookup) } @@ -245,7 +243,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_vertex_api_key(api_key, env_lookup) } @@ -264,7 +262,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { let mut data = Map::new(); data.insert( "model".to_string(), @@ -289,10 +287,10 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; @@ -314,7 +312,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { }); } - let object = ocr_data.as_object().ok_or_else(|| CoreError::InvalidType { + let object = ocr_data.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&ocr_data), })?; @@ -346,7 +344,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { _model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_vertex_deepseek_url(api_base, optional_params, env_lookup) } @@ -354,7 +352,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_vertex_api_key(api_key, env_lookup) } } diff --git a/litellm-rust/crates/core/src/realtime/transformation.rs b/litellm-rust/crates/core/src/realtime/transformation.rs index 69b88687000..b08084514ef 100644 --- a/litellm-rust/crates/core/src/realtime/transformation.rs +++ b/litellm-rust/crates/core/src/realtime/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; pub trait RealtimeProviderConfig { @@ -11,12 +11,12 @@ pub trait RealtimeProviderConfig { &self, event: &RealtimeEvent, model: &str, - ) -> CoreResult; + ) -> Result; /// Transform a backend → client event before it is forwarded downstream. fn transform_realtime_response( &self, event: &RealtimeEvent, model: &str, - ) -> CoreResult; + ) -> Result; } diff --git a/litellm-rust/crates/core/src/responses/instrumentation.rs b/litellm-rust/crates/core/src/responses/instrumentation.rs index ec04571da14..b1098f4d386 100644 --- a/litellm-rust/crates/core/src/responses/instrumentation.rs +++ b/litellm-rust/crates/core/src/responses/instrumentation.rs @@ -5,9 +5,9 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde_json::Value; +use crate::Error; use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType}; -use crate::{CoreError, CoreResult}; #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ResponsesWsUsage { @@ -205,7 +205,7 @@ impl ResponsesWsInstrumentation { } } -type LifecycleFuture<'a, T> = Pin> + Send + 'a>>; +type LifecycleFuture<'a, T> = Pin> + Send + 'a>>; impl CallLifecycleHooks<(), (), ()> for ResponsesWsInstrumentation { type PreCallFuture<'a> = LifecycleFuture<'a, ()>; @@ -246,7 +246,7 @@ impl CallLifecycleHooks<(), (), ()> for ResponsesWsInstrumentation { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -342,7 +342,7 @@ mod tests { ), (), &instrumentation, - |_| async { Ok::<(), CoreError>(()) }, + |_| async { Ok::<(), Error>(()) }, ) .await; diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/core/src/responses/websocket.rs index 92dc19627a0..5d037e9cf1b 100644 --- a/litellm-rust/crates/core/src/responses/websocket.rs +++ b/litellm-rust/crates/core/src/responses/websocket.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}; use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult}; @@ -19,13 +19,13 @@ pub trait ResponsesWebSocketProviderConfig: Sync { &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult; + ) -> Result; fn transform_ws_response( &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult; + ) -> Result; } pub fn complete_websocket_url( diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 746e0770f9b..68aa9436b15 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -10,7 +10,7 @@ use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompleti use litellm_core::chat_completions::{ chat_completions as run_chat_completions, chat_completions_decline_reason, }; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::messages::messages as run_messages; use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; use litellm_python_interop::{from_py, release_count, release_gil, to_py}; @@ -54,13 +54,13 @@ fn chat_completions_response_to_py( to_py(py, &response) } -fn core_error_to_pyerr(err: CoreError) -> PyErr { +fn core_error_to_pyerr(err: Error) -> PyErr { match err { - CoreError::Auth(message) => PyValueError::new_err(message), - CoreError::InvalidProvider(_) - | CoreError::InvalidRequest(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) => PyValueError::new_err(err.to_string()), + Error::Auth(message) => PyValueError::new_err(message), + Error::InvalidProvider(_) + | Error::InvalidRequest(_) + | Error::InvalidType { .. } + | Error::MissingField(_) => PyValueError::new_err(err.to_string()), other => PyRuntimeError::new_err(other.to_string()), } } @@ -71,22 +71,22 @@ fn core_error_to_pyerr(err: CoreError) -> PyErr { /// Everything raised before the request goes out is safe for the host to retry /// on its own path; anything after it is not, because the provider has already /// done the work and billed for it. -fn chat_completions_error_to_pyerr(err: CoreError) -> PyErr { +fn chat_completions_error_to_pyerr(err: Error) -> PyErr { match err { - CoreError::Unsupported(_) - | CoreError::Auth(_) - | CoreError::InvalidProvider(_) - | CoreError::InvalidRequest(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) - | CoreError::Routing(_) + Error::Unsupported(_) + | Error::Auth(_) + | Error::InvalidProvider(_) + | Error::InvalidRequest(_) + | Error::InvalidType { .. } + | Error::MissingField(_) + | Error::Routing(_) // Nothing reached the provider, so serving it on Python cannot double // bill and is the only way the caller gets an answer at all. - | CoreError::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), - CoreError::Http { status, body } => { + | Error::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), + Error::Http { status, body } => { RustUpstreamError::new_err((status, format!("{status}: {body}"))) } - CoreError::Network(message) | CoreError::InvalidResponse(message) => { + Error::Network(message) | Error::InvalidResponse(message) => { RustUpstreamError::new_err((0u16, message)) } } From 9bd870d47a700b183e0ee0d9bf7647aa7c739561 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:36:28 -0700 Subject: [PATCH 103/175] fix(databricks): upgrade legacy thinking to adaptive on adaptive-only Claude models --- .../llms/databricks/chat/transformation.py | 4 ++++ .../test_databricks_chat_transformation.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index c587146005f..65622d62af2 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -330,6 +330,10 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): ) -> dict: is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params) mapped_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params) + if "claude" in model: + AnthropicConfig.translate_legacy_thinking_for_adaptive_model( + model=model, optional_params=mapped_params, custom_llm_provider="databricks" + ) if "tools" in mapped_params: mapped_params["tools"] = self._map_openai_to_dbrx_tool(model=model, tools=mapped_params["tools"]) if "max_completion_tokens" in non_default_params and replace_max_completion_tokens_with_max_tokens: diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index 41fb2589655..71661cc532b 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -422,6 +422,27 @@ def test_databricks_config_probes_capabilities_under_databricks_namespace(): assert DatabricksConfig().custom_llm_provider == "databricks" +@pytest.mark.parametrize( + "model, expected_thinking, expected_output_config", + [ + ("databricks-claude-opus-4-8", {"type": "adaptive"}, {"effort": "high"}), + ("databricks-claude-opus-4-6", {"type": "enabled", "budget_tokens": 4096}, None), + ], + ids=["adaptive_only_upgrades_to_adaptive", "legacy_capable_forwards_verbatim"], +) +def test_map_openai_params_upgrades_legacy_thinking_on_adaptive_only_claude( + model, expected_thinking, expected_output_config +): + mapped = DatabricksConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 4096}}, + optional_params={}, + model=model, + drop_params=False, + ) + assert mapped["thinking"] == expected_thinking + assert mapped.get("output_config") == expected_output_config + + def _streaming_chunk(usage=None, choices=None): base = { "id": "chatcmpl-test", From 3af19cbf61706f1cf3d3360b8802b5cbc1991827 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:36:29 -0700 Subject: [PATCH 104/175] test(proxy-extras): use the modern optional annotation in the deploy budget test --- tests/proxy_migration_tests/test_prisma_toolchain.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/proxy_migration_tests/test_prisma_toolchain.py b/tests/proxy_migration_tests/test_prisma_toolchain.py index 4c258c4d007..733870f3239 100644 --- a/tests/proxy_migration_tests/test_prisma_toolchain.py +++ b/tests/proxy_migration_tests/test_prisma_toolchain.py @@ -22,7 +22,6 @@ import sys import time from collections.abc import Callable from pathlib import Path -from typing import Optional import pytest @@ -302,7 +301,7 @@ def test_db_push_timeout_hint_names_the_per_command_budget( ids=["raised_command_budget_carries_over", "lowered_command_budget_does_not", "override_wins_upward", "override_wins_downward"], ) def test_migrate_deploy_budget_keeps_a_raised_command_budget( - command_timeout: str, deploy_timeout: Optional[str], expected: float, monkeypatch: pytest.MonkeyPatch + command_timeout: str, deploy_timeout: str | None, expected: float, monkeypatch: pytest.MonkeyPatch ) -> None: """Deployments that raised the per-command budget to survive a long deploy keep that budget for deploy.""" monkeypatch.setenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, command_timeout) From 034ff5855802e1b2b036d4cee6208a3efd8a15fd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:36:32 -0700 Subject: [PATCH 105/175] test(otel): assert Langfuse logger behavior instead of its class --- .../integrations/otel/test_langfuse_logger.py | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/tests/test_litellm/integrations/otel/test_langfuse_logger.py b/tests/test_litellm/integrations/otel/test_langfuse_logger.py index af0597517dc..3e35395389e 100644 --- a/tests/test_litellm/integrations/otel/test_langfuse_logger.py +++ b/tests/test_litellm/integrations/otel/test_langfuse_logger.py @@ -12,9 +12,9 @@ pytest.importorskip("opentelemetry") from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +import litellm # noqa: E402 from litellm.caching.dual_cache import DualCache # noqa: E402 -from litellm.integrations.otel.langfuse_logger import LangfuseOpenTelemetryV2 # noqa: E402 -from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger # noqa: E402 +from litellm.integrations.otel.logger import build_otel_v2_logger # noqa: E402 from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402 from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402 from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402 @@ -22,6 +22,7 @@ from litellm.integrations.otel.plumbing import providers # noqa: E402 from litellm.integrations.otel.plumbing.context import set_request_root_span # noqa: E402 from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 # noqa: E402 from litellm.proxy._types import UserAPIKeyAuth # noqa: E402 +from litellm.proxy.utils import ProxyLogging # noqa: E402 from litellm.types.llms.openai import ( # noqa: E402 ResponseCompletedEvent, ResponsesAPIResponse, @@ -294,13 +295,29 @@ def test_unrenderable_output_never_raises_into_the_request(): def test_factory_keeps_the_base_logger_unless_langfuse_content_capture_is_on(capture, mappers): logger, exporter = _logger(capture=capture, mappers=mappers) - assert type(logger) is OpenTelemetryV2 _run_request(logger, CHAT_DATA, "acompletion", ModelResponse()) attrs = _root_attrs(exporter) assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs -def test_langfuse_otel_preset_builds_the_langfuse_logger(monkeypatch): +@pytest.mark.parametrize( + ("capture", "mappers", "relays_streams"), + [ + ("span_only", ("genai", "langfuse"), True), + ("no_content", ("genai", "langfuse"), False), + ("span_only", ("genai",), False), + ], +) +def test_only_langfuse_content_capture_takes_proxy_streams_off_the_fast_path( + monkeypatch, capture, mappers, relays_streams +): + logger, _ = _logger(capture=capture, mappers=mappers) + monkeypatch.setattr(litellm, "callbacks", [logger]) + + assert ProxyLogging._callback_capabilities().has_iterator_override is relays_streams + + +def test_langfuse_otel_preset_builds_a_logger_that_stamps_the_root(monkeypatch): monkeypatch.setenv("LITELLM_OTEL_V2", "true") monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk") monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk") @@ -311,7 +328,10 @@ def test_langfuse_otel_preset_builds_the_langfuse_logger(monkeypatch): loggers: list = [] try: built = _maybe_construct_otel_v2("langfuse_otel", loggers) - assert isinstance(built, LangfuseOpenTelemetryV2) + assert built is not None assert _maybe_construct_otel_v2("langfuse_otel", loggers) is built + root = _start_root(built) + asyncio.run(built.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) + assert INPUT_ATTR in dict(root.attributes or {}) finally: is_otel_v2_enabled.cache_clear() From cde9d94c3650dfbc0b704280973d6e4319a7530a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:40:24 -0700 Subject: [PATCH 106/175] feat(agentcore-a2a): derive runtime session id from A2A message.contextId (#39371) Native AgentCore A2A always sent either a fresh generated runtime session id or the single configured runtimeSessionId, so related turns lost context and unrelated callers shared one AgentCore microVM. The runtime session id is now params.message.contextId scoped to the calling key hash, then runtimeSessionId, then generated, and is length-validated (33-256) before the header is signed. Invalid ids surface as JSON-RPC -32602 / HTTP 400 instead of a 500. Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../bedrock_agentcore/transformation.py | 51 ++++- litellm/a2a_protocol/utils.py | 25 +++ litellm/llms/langflow/a2a.py | 27 +-- .../proxy/agent_endpoints/a2a_endpoints.py | 2 + .../test_bedrock_agentcore_a2a.py | 191 ++++++++++++++++++ .../agent_endpoints/test_a2a_endpoints.py | 56 +++++ 6 files changed, 325 insertions(+), 27 deletions(-) diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py index 32252711997..1e8cc4ff90e 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py @@ -10,8 +10,19 @@ from collections.abc import AsyncIterator, Mapping from typing import Any, Final from litellm._logging import verbose_logger +from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, +) +from litellm.a2a_protocol.utils import ( + get_session_id_from_a2a_params, + scope_session_to_principal, +) +from litellm.exceptions import BadRequestError from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig +RUNTIME_SESSION_ID_MIN_LENGTH: Final = 33 +RUNTIME_SESSION_ID_MAX_LENGTH: Final = 256 + # Reserved outbound header names that must never be sourced from per-request # ``agent_extra_headers`` for AgentCore requests. ``agent_extra_headers`` carries # values rewritten from the client-controlled ``x-a2a-{agent}-*`` convention, so @@ -19,8 +30,9 @@ from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreCo # request identity / SigV4 metadata by overwriting headers the proxy sets from # trusted server-side config. # -# The runtime headers (session / user id) are derived server-side from -# ``runtimeSessionId`` / ``runtimeUserId`` in the agent's ``litellm_params``; +# The runtime headers (session / user id) are derived server-side from the A2A +# ``message.contextId`` and ``runtimeSessionId`` / ``runtimeUserId`` in the +# agent's ``litellm_params``; # ``authorization`` is set by the AgentCore signer (JWT or SigV4); ``host`` and # the ``x-amz-*`` family are owned by SigV4 itself. _RESERVED_EXACT_HEADERS: Final = frozenset( @@ -66,6 +78,31 @@ def _filter_reserved_headers( return filtered or None +def _request_scoped_runtime_session_id( + params: Mapping[str, Any], + litellm_params: Mapping[str, Any], +) -> str | None: + context_id: Final = get_session_id_from_a2a_params(params) + if not isinstance(context_id, str) or not context_id: + return None + return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)) + + +def _validate_runtime_session_id(session_id: str, model: str) -> str: + if RUNTIME_SESSION_ID_MIN_LENGTH <= len(session_id) <= RUNTIME_SESSION_ID_MAX_LENGTH: + return session_id + raise BadRequestError( + message=( + f"Invalid AgentCore runtime session id {session_id!r}: AWS requires " + f"{RUNTIME_SESSION_ID_MIN_LENGTH}-{RUNTIME_SESSION_ID_MAX_LENGTH} characters. It is built from the A2A " + "message.contextId (prefixed with a 16-hex-char hash of the calling key and '-') when set, " + "otherwise from the agent's configured runtimeSessionId." + ), + model=model, + llm_provider="bedrock", + ) + + class BedrockAgentCoreA2ATransformation: """ Request/response transformation for Bedrock AgentCore A2A agents. @@ -100,7 +137,9 @@ class BedrockAgentCoreA2ATransformation: here to prevent a caller-controlled ``x-a2a-{agent}-*`` header from spoofing the AgentCore runtime user id or other SigV4 metadata. Use ``api_key`` / ``runtimeUserId`` / ``runtimeSessionId`` in litellm_params - (not ``agent_extra_headers``) to override those values. + (not ``agent_extra_headers``) to override those values. The runtime + session id is taken from ``params["message"]["contextId"]`` (scoped to + the calling key) when present, then ``runtimeSessionId``, else generated. Returns: Tuple of (url, signed_headers, signed_body_bytes) @@ -139,7 +178,11 @@ class BedrockAgentCoreA2ATransformation: # Set required AgentCore session headers (normally set by transform_request, # which we skip because it also builds {"prompt": "..."}) headers: Final[dict] = {} - session_id: Final = agentcore_config._get_runtime_session_id(optional_params) + session_id: Final = _validate_runtime_session_id( + _request_scoped_runtime_session_id(params, litellm_params) + or agentcore_config._get_runtime_session_id(optional_params), + model=model, + ) headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = session_id runtime_user_id: Final = agentcore_config._get_runtime_user_id(optional_params) if runtime_user_id: diff --git a/litellm/a2a_protocol/utils.py b/litellm/a2a_protocol/utils.py index f2e61f66105..7c459daf720 100644 --- a/litellm/a2a_protocol/utils.py +++ b/litellm/a2a_protocol/utils.py @@ -2,6 +2,8 @@ Utility functions for A2A protocol. """ +import hashlib +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final import litellm @@ -140,6 +142,29 @@ class A2ARequestUtils: return prompt_tokens, completion_tokens, total_tokens +def get_session_id_from_a2a_params(params: Mapping[str, Any]) -> str | None: + message: Final = params.get("message", {}) + if isinstance(message, dict): + return message.get("contextId") + return getattr(message, "contextId", None) + + +def scope_session_to_principal(session_id: str, principal: str | None) -> str: + """ + Bind a client-supplied A2A contextId to the authenticated principal. + + Without this, two distinct keys authorized for the same agent could set the + same contextId and read/append to each other's backend memory. The + principal is hashed (it is already a hashed token) so the raw value is never + sent to the agent backend, while the original contextId is kept as a suffix + for operator-side correlation. + """ + if not principal: + return session_id + principal_prefix: Final = hashlib.sha256(principal.encode("utf-8")).hexdigest()[:16] + return f"{principal_prefix}-{session_id}" + + # Backwards compatibility aliases def extract_text_from_a2a_message(message: Any) -> str: return A2ARequestUtils.extract_text_from_message(message) diff --git a/litellm/llms/langflow/a2a.py b/litellm/llms/langflow/a2a.py index cae750d586e..060dc0a4d05 100644 --- a/litellm/llms/langflow/a2a.py +++ b/litellm/llms/langflow/a2a.py @@ -1,28 +1,9 @@ -import hashlib from typing import Any, Final - -def get_session_id_from_a2a_params(params: dict[str, Any]) -> str | None: - message: Final = params.get("message", {}) - if isinstance(message, dict): - return message.get("contextId") - return getattr(message, "contextId", None) - - -def scope_session_to_principal(session_id: str, principal: str | None) -> str: - """ - Bind a client-supplied A2A contextId to the authenticated principal. - - Without this, two distinct keys authorized for the same LangFlow agent could - set the same contextId and read/append to each other's LangFlow memory. The - principal is hashed (it is already a hashed token) so the raw value is never - sent to the LangFlow backend, while the original contextId is kept as a - suffix for operator-side correlation. - """ - if not principal: - return session_id - principal_prefix: Final = hashlib.sha256(principal.encode("utf-8")).hexdigest()[:16] - return f"{principal_prefix}-{session_id}" +from litellm.a2a_protocol.utils import ( + get_session_id_from_a2a_params, + scope_session_to_principal, +) def merge_a2a_session_into_litellm_params( diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 31b05320cd3..28882484db4 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -1019,4 +1019,6 @@ async def invoke_agent_a2a( ) except Exception: pass + if isinstance(e, litellm.BadRequestError): + return _jsonrpc_error(body.get("id"), -32602, e.message, 400) return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {e}", 500) diff --git a/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py index 5503a5668bf..a8fe464ec32 100644 --- a/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py +++ b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py @@ -11,7 +11,9 @@ Verifies that: import json +import httpx import pytest +import respx from unittest.mock import AsyncMock, MagicMock, patch @@ -295,6 +297,195 @@ class TestTransformation: assert headers["Authorization"].startswith("AWS4-HMAC-SHA256") +SESSION_HEADER = "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id" +CONTEXT_ID = "conversation-alpha-0001-0000000000000000" +KEY_HASH = "hashed-key-of-caller-one" + + +def _params_with_context(context_id: object) -> dict: + return {"message": {**SAMPLE_PARAMS["message"], "contextId": context_id}} + + +def _scoped(context_id: str, key_hash: str) -> str: + import hashlib + + return f"{hashlib.sha256(key_hash.encode()).hexdigest()[:16]}-{context_id}" + + +def _session_header(params: dict, litellm_params: dict) -> str: + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=params, + litellm_params=litellm_params, + ) + return headers[SESSION_HEADER] + + +@pytest.fixture +def httpx_transport(monkeypatch): + import litellm + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +class TestRequestScopedRuntimeSession: + """message.contextId selects the AgentCore runtime session, scoped to the calling key.""" + + def test_context_id_scoped_to_calling_key(self): + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, + ) + + litellm_params = {**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: KEY_HASH} + assert _session_header(_params_with_context(CONTEXT_ID), litellm_params) == _scoped(CONTEXT_ID, KEY_HASH) + + def test_context_id_used_verbatim_without_principal(self): + assert _session_header(_params_with_context(CONTEXT_ID), SAMPLE_LITELLM_PARAMS) == CONTEXT_ID + + def test_same_context_id_reuses_session_and_other_context_isolated(self): + first = _session_header(_params_with_context(CONTEXT_ID), SAMPLE_LITELLM_PARAMS) + second = _session_header(_params_with_context(CONTEXT_ID), SAMPLE_LITELLM_PARAMS) + other = _session_header( + _params_with_context("conversation-beta-00002-0000000000000000"), + SAMPLE_LITELLM_PARAMS, + ) + assert first == second + assert other != first + + def test_same_context_id_from_different_keys_is_isolated(self): + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, + ) + + params = _params_with_context(CONTEXT_ID) + caller_one = _session_header(params, {**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: KEY_HASH}) + caller_two = _session_header( + params, {**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: "hashed-key-of-caller-two"} + ) + assert caller_one != caller_two + assert caller_one.endswith(f"-{CONTEXT_ID}") + assert caller_two.endswith(f"-{CONTEXT_ID}") + + def test_context_id_takes_precedence_over_configured_session(self): + litellm_params = {**SAMPLE_LITELLM_PARAMS, "runtimeSessionId": "a" * 40} + assert _session_header(_params_with_context(CONTEXT_ID), litellm_params) == CONTEXT_ID + + def test_configured_session_is_fallback_without_context_id(self): + litellm_params = {**SAMPLE_LITELLM_PARAMS, "runtimeSessionId": "a" * 40} + assert _session_header(SAMPLE_PARAMS, litellm_params) == "a" * 40 + assert _session_header(_params_with_context(""), litellm_params) == "a" * 40 + + def test_no_context_id_and_no_config_generates_new_session_per_request(self): + first = _session_header(SAMPLE_PARAMS, SAMPLE_LITELLM_PARAMS) + second = _session_header(SAMPLE_PARAMS, SAMPLE_LITELLM_PARAMS) + assert first != second + assert 33 <= len(first) <= 256 + + @pytest.mark.parametrize( + "context_id", + [ + "short-context-id", + "x" * 257, + ], + ) + def test_invalid_context_id_rejected_with_clear_error(self, context_id): + import litellm + + with pytest.raises(litellm.BadRequestError, match="Invalid AgentCore runtime session id") as exc_info: + _session_header(_params_with_context(context_id), SAMPLE_LITELLM_PARAMS) + assert exc_info.value.status_code == 400 + assert "33-256" in str(exc_info.value) + + def test_scoped_context_id_shorter_than_33_rejected(self): + import litellm + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, + ) + + litellm_params = {**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: KEY_HASH} + with pytest.raises(litellm.BadRequestError, match=_scoped("c" * 15, KEY_HASH)): + _session_header(_params_with_context("c" * 15), litellm_params) + assert _session_header(_params_with_context("c" * 16), litellm_params) == _scoped("c" * 16, KEY_HASH) + + def test_invalid_configured_session_rejected(self): + import litellm + + litellm_params = {**SAMPLE_LITELLM_PARAMS, "runtimeSessionId": "too-short"} + with pytest.raises(litellm.BadRequestError, match="Invalid AgentCore runtime session id"): + _session_header(SAMPLE_PARAMS, litellm_params) + + def test_non_string_context_id_falls_back(self): + litellm_params = {**SAMPLE_LITELLM_PARAMS, "runtimeSessionId": "a" * 40} + assert _session_header(_params_with_context(12345), litellm_params) == "a" * 40 + + def test_spoofed_session_header_does_not_override_context_id(self): + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=_params_with_context(CONTEXT_ID), + litellm_params=SAMPLE_LITELLM_PARAMS, + agent_extra_headers={SESSION_HEADER: "s" * 40}, + ) + assert headers[SESSION_HEADER] == CONTEXT_ID + + @pytest.mark.asyncio + async def test_context_id_session_header_on_outbound_non_streaming_post(self, httpx_transport): + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, + ) + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + with respx.mock(assert_all_called=True) as router: + route = router.post(url__regex=r".*/invocations.*").mock( + return_value=httpx.Response(200, json={"jsonrpc": "2.0", "id": "req-001", "result": {}}) + ) + await BedrockAgentCoreA2AConfig().handle_non_streaming( + request_id="req-001", + params=_params_with_context(CONTEXT_ID), + litellm_params={**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: KEY_HASH}, + ) + + assert route.calls.last.request.headers[SESSION_HEADER] == _scoped(CONTEXT_ID, KEY_HASH) + + @pytest.mark.asyncio + async def test_context_id_session_header_on_outbound_streaming_post(self, httpx_transport): + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, + ) + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + with respx.mock(assert_all_called=True) as router: + route = router.post(url__regex=r".*/invocations.*").mock( + return_value=httpx.Response(200, json={"jsonrpc": "2.0", "id": "req-001", "result": {}}) + ) + events = [ + event + async for event in BedrockAgentCoreA2AConfig().handle_streaming( + request_id="req-001", + params=_params_with_context(CONTEXT_ID), + litellm_params={**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: KEY_HASH}, + ) + ] + + assert events == [{"jsonrpc": "2.0", "id": "req-001", "result": {}}] + assert route.calls.last.request.headers[SESSION_HEADER] == _scoped(CONTEXT_ID, KEY_HASH) + + class TestNonStreaming: """Test end-to-end non-streaming flow.""" diff --git a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py index 2ff38af80b1..43034f889f6 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py @@ -918,6 +918,62 @@ async def test_task_method_failure_hook_uses_enriched_request_data(): assert failure_data.get("agent_id") == "test-agent" +@pytest.mark.asyncio +async def test_agentcore_invalid_context_id_returns_jsonrpc_invalid_params_400(): + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + agent.litellm_params = { + "custom_llm_provider": "bedrock", + "model": "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/demo", + "api_key": "test-jwt-token", + } + mock_request = _make_request_mock( + "message/send", + { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "Hello"}], + "messageId": "msg-1", + "contextId": "too-short", + } + }, + ) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + mock_proxy_logging = MagicMock() + mock_proxy_logging.pre_call_hook = AsyncMock( + side_effect=lambda user_api_key_dict, data, call_type: data + ) + mock_proxy_logging.post_call_failure_hook = AsyncMock(return_value=None) + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( # test-quality-ok: same proxy_logging_obj injection the sibling failure-hook test uses; no HTTP call is made because the request is rejected before signing + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + body = json.loads(response.body.decode()) + assert response.status_code == 400 + assert body["id"] == "req-1" + assert body["error"]["code"] == -32602 + assert "Invalid AgentCore runtime session id" in body["error"]["message"] + assert "Internal error" not in body["error"]["message"] + mock_proxy_logging.post_call_failure_hook.assert_awaited_once() + + @pytest.mark.asyncio async def test_get_extended_agent_card_rewrites_url(): from litellm.proxy._types import UserAPIKeyAuth From 987ab769213527d9bd1cfaeb3b038674dbec40f4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:40:59 -0700 Subject: [PATCH 107/175] fix(proxy): share per-model budget counters across replicas through the spend counter cache (#39375) * fix(proxy): share per-model budget counters across replicas through the spend counter cache Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): keep the shared fake Redis store immutable Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/model_max_budget_limiter.py | 35 +++++--- litellm/proxy/proxy_server.py | 2 +- ...test_unit_test_max_model_budget_limiter.py | 85 +++++++++++++++++++ .../proxy/test_redis_auth_cache_flag.py | 24 +++++- 4 files changed, 132 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index c5d10b2749b..efaaab277a9 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -1,6 +1,6 @@ import json import time -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import Final @@ -199,18 +199,10 @@ async def build_model_max_budget_usage( ) for budget_model, budget_config in budgets ) - batched: Final = await cache.async_batch_get_cache( - keys=list(spend_keys) # mutable-ok: async_batch_get_cache annotates keys as list, so one must exist here - ) - # async_batch_get_cache returns None if it fails internally, and its result is - # index-aligned with `keys` otherwise. An unusable result reads as a miss, - # which is what a never-written counter already reads as. - current_spends: Final = ( - tuple(batched) if isinstance(batched, list) and len(batched) == len(budgets) else (None,) * len(budgets) - ) + current_spends: Final = await _current_window_spends(cache=cache, spend_keys=spend_keys) return { budget_model: { - "current_spend": round(_as_spend(current_spend), 4), + "current_spend": round(current_spend, 4), "budget_limit": budget_config.max_budget, "time_period": budget_config.budget_duration, } @@ -218,6 +210,22 @@ async def build_model_max_budget_usage( } +async def _current_window_spends(cache: DualCache, spend_keys: Sequence[str]) -> tuple[float, ...]: + """Redis holds the window total across replicas; the in-memory copy is one replica's share.""" + keys: Final = list(spend_keys) # mutable-ok: both batch readers annotate their key argument as list + redis_cache: Final = cache.redis_cache + if redis_cache is not None: + shared: Final = await redis_cache.async_batch_get_cache(key_list=keys) + return tuple(_as_spend(shared.get(key)) for key in keys) + # async_batch_get_cache returns None if it fails internally, and its result is + # index-aligned with `keys` otherwise. An unusable result reads as a miss, + # which is what a never-written counter already reads as. + batched: Final = await cache.async_batch_get_cache(keys=keys) + if not isinstance(batched, list) or len(batched) != len(keys): + return (0.0,) * len(keys) + return tuple(_as_spend(current_spend) for current_spend in batched) + + def _usable_budget_config(raw_budget_config: object) -> BudgetConfig | None: try: budget_config: Final = BudgetConfig.model_validate(raw_budget_config) @@ -404,7 +412,10 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): return current_spend + _as_spend(await self._cached_spend(legacy_spend_key)) async def _cached_spend(self, spend_key: str) -> float | None: - return await self.dual_cache.async_get_cache(key=spend_key) + redis_cache: Final = self.dual_cache.redis_cache + if redis_cache is None: + return await self.dual_cache.async_get_cache(key=spend_key) + return await redis_cache.async_get_cache(key=spend_key) async def async_filter_deployments( self, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 85a57e5af2b..d52f05f6166 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2280,7 +2280,7 @@ user_api_key_cache: UserApiKeyCache = UserApiKeyCache( ) spend_counter_cache: Final = DualCache(default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value) cli_sso_session_cache: Final = DualCache(default_in_memory_ttl=CLI_SSO_SESSION_TTL_SECONDS) -model_max_budget_limiter: Final = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=user_api_key_cache) +model_max_budget_limiter: Final = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=spend_counter_cache) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) redis_usage_cache: RedisCache | None = None # redis cache used for tracking spend, tpm/rpm limits polling_via_cache_enabled: Literal["all"] | list[str] | bool = False diff --git a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py index 3785ccdcfba..096efc33aaf 100644 --- a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py +++ b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py @@ -1,3 +1,5 @@ +import asyncio +from types import MappingProxyType from unittest.mock import AsyncMock, patch @@ -5,6 +7,7 @@ import pytest import litellm from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCache from datetime import datetime, timezone from litellm.litellm_core_utils.duration_parser import duration_in_seconds @@ -1332,3 +1335,85 @@ async def test_the_user_scope_has_no_pre_upgrade_counter_to_carry(): await limiter.is_user_within_model_budget( user_id="u1", user_model_max_budget=model_max_budget, model="openai/gpt-4" ) + + +class _SharedFakeRedis(RedisCache): + """Stand-in for the one Redis every replica's DualCache is attached to. + + Only the methods the limiter and DualCache call are implemented, and + ``super().__init__`` is skipped so no connection is opened. + """ + + def __init__(self): + self._store = MappingProxyType({}) + + async def async_set_cache(self, key, value, **kwargs): + self._store = MappingProxyType({**self._store, key: value}) + + async def async_get_cache(self, key, **kwargs): + return self._store.get(key) + + async def async_batch_get_cache(self, key_list, **kwargs): + return {key: self._store.get(key) for key in key_list} + + async def async_increment_pipeline(self, increment_list, **kwargs): + for op in increment_list: + total = self._store.get(op["key"], 0.0) + op["increment_value"] + self._store = MappingProxyType({**self._store, op["key"]: total}) + return [self._store[op["key"]] for op in increment_list] + + +async def _log_spend(limiter, *, key_hash, model_max_budget, response_cost): + await limiter.async_log_success_event( + _success_kwargs( + model_group="gpt-4", + response_cost=response_cost, + key_hash=key_hash, + key_model_max_budget=model_max_budget, + ), + response_obj=None, + start_time=None, + end_time=None, + ) + # The Redis push is scheduled as a task rather than awaited inline. + await asyncio.gather(*(t for t in asyncio.all_tasks() if t is not asyncio.current_task())) + + +@pytest.mark.asyncio +async def test_spend_logged_on_one_replica_is_enforced_and_reported_on_another(): + """ + Each replica increments its own in-memory copy of the per-model counter and + pushes the increment to the shared Redis, so only Redis holds the window's + total. A replica that has served part of the traffic must still enforce and + report the total, not its own share. + + Regression: reads went to the in-memory tier first, so a replica whose local + copy sat under the cap kept admitting requests and /key/info on it reported + that local share, while the shared counter was already over the cap. + """ + shared_redis = _SharedFakeRedis() + replica_a = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache(redis_cache=shared_redis)) + replica_b = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache(redis_cache=shared_redis)) + key_hash = "vk-shared" + model_max_budget = {"gpt-4": {"budget_limit": 1.0, "time_period": "30d"}} + user_api_key = UserAPIKeyAuth(token=key_hash, model_max_budget=model_max_budget) + + await _log_spend(replica_b, key_hash=key_hash, model_max_budget=model_max_budget, response_cost=0.25) + await _log_spend(replica_a, key_hash=key_hash, model_max_budget=model_max_budget, response_cost=0.5) + await _log_spend(replica_a, key_hash=key_hash, model_max_budget=model_max_budget, response_cost=0.5) + + with pytest.raises(litellm.BudgetExceededError): + await replica_b.is_key_within_model_budget(user_api_key, "gpt-4") + + usage_on_b = await build_model_max_budget_usage( + entity_type=Litellm_EntityType.KEY, + entity_id=key_hash, + model_max_budget=model_max_budget, + cache=replica_b.dual_cache, + ) + assert usage_on_b["gpt-4"]["current_spend"] == 1.25 + + # Control: a replica that never served this key reads the same total. + replica_c = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache(redis_cache=shared_redis)) + with pytest.raises(litellm.BudgetExceededError): + await replica_c.is_key_within_model_budget(user_api_key, "gpt-4") diff --git a/tests/test_litellm/proxy/test_redis_auth_cache_flag.py b/tests/test_litellm/proxy/test_redis_auth_cache_flag.py index 772b08bc9d0..573bfc40c96 100644 --- a/tests/test_litellm/proxy/test_redis_auth_cache_flag.py +++ b/tests/test_litellm/proxy/test_redis_auth_cache_flag.py @@ -5,7 +5,7 @@ Verifies that _init_cache attaches Redis to user_api_key_cache only when the flag is explicitly set to True, and leaves it in-memory-only otherwise. """ -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager import json from unittest.mock import MagicMock, patch @@ -167,3 +167,25 @@ class TestRedisAuthCacheFlag: f"cli_sso_session_cache must always get Redis " f"(enable_redis_auth_cache={flag_value!r})" ) + + def test_flag_absent_still_shares_the_model_budget_counters_over_redis(self): + """ + Per-model budget counters are spend counters: the limiter must be able to + push and read them through Redis without the auth-cache opt-in, or every + worker enforces and reports its own share of a key's spend + """ + fake_redis = _FakeRedisCache() + limiter_cache = ps.model_max_budget_limiter.dual_cache + touched_caches = ( + limiter_cache, + ps.spend_counter_cache, + ps.cli_sso_session_cache, + ps.user_api_key_cache, + ps.litellm_config_cache, + ) + with ExitStack() as detached: + for cache in touched_caches: + detached.enter_context(patch.object(cache, "redis_cache", None)) + ps._attach_redis_usage_cache(fake_redis, enable_redis_auth_cache=False) + assert limiter_cache.redis_cache is fake_redis + assert ps.user_api_key_cache.redis_cache is None From dc12e4c2b4ad31b1eda1544ecd2e424aabc78151 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:57:01 -0700 Subject: [PATCH 108/175] fix(responses): match guardrail tools by ordinal in one pass Sort the chat-tool keys once and number duplicates with groupby instead of rescanning every preceding key per position, so the guardrail merge stays O(n log n) on client-supplied tool lists. Drop the comment that restated the unsupported-tool warning in the Responses-to-chat transformation. --- .../guardrail_translation/tool_merge.py | 8 ++++++-- .../transformation.py | 4 ---- ...st_openai_responses_guardrail_tool_merge.py | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/tool_merge.py b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py index 3ae951d3f61..9fbf9f31a0f 100644 --- a/litellm/llms/openai/responses/guardrail_translation/tool_merge.py +++ b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py @@ -1,5 +1,5 @@ from collections.abc import Iterable, Mapping, Sequence -from itertools import accumulate, chain +from itertools import accumulate, chain, groupby from types import MappingProxyType from typing import Final, TypeAlias @@ -47,7 +47,11 @@ def _chat_tool_key(tool: Tool) -> str: def _indexed_keys(tools: Sequence[Tool]) -> tuple[IndexedKey, ...]: keys: Final = tuple(_chat_tool_key(tool) for tool in tools) - return tuple((key, keys[:position].count(key)) for position, key in enumerate(keys)) + positions_by_key: Final = groupby(sorted(range(len(keys)), key=keys.__getitem__), key=keys.__getitem__) + ordinal_by_position: Final = MappingProxyType( + {position: ordinal for _, positions in positions_by_key for ordinal, position in enumerate(positions)} + ) + return tuple((key, ordinal_by_position[position]) for position, key in enumerate(keys)) def _namespace_members(namespace: Tool) -> tuple[Tool, ...]: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 9f91d7527cb..b2d1a69e0d8 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1903,10 +1903,6 @@ class LiteLLMCompletionResponsesConfig: converted: Final = convert_custom_tool_to_function_tool(tool) return ResponsesToolChatForm(chat_tools=() if converted is None else (converted,), web_search_options=None) if tool_type in ("computer_use", "image_generation", "shell"): - # Drop unsupported Responses-API-only tool types that have no - # Chat Completions equivalent. Passing them through verbatim - # causes providers to reject the request with "'function' is a - # required property". verbose_logger.warning( "Dropping Responses API tool of type '%s': it has no Chat Completions " "equivalent and the target provider would reject the request.", diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py index b80dd0b36aa..4075c209606 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py @@ -56,6 +56,24 @@ def test_duplicate_function_names_are_matched_by_ordinal(): assert list(merged) == [original[0]] +def test_interleaved_duplicate_names_keep_their_own_ordinals(): + original = [ + _function("dup", "a"), + _function("other", "x"), + _function("dup", "b"), + _function("dup", "c"), + _function("other", "y"), + ] + groups = _groups(original) + flat = _flat(groups) + edited = {**flat[3], "function": {**flat[3]["function"], "description": "changed"}} + + merged = merge_guardrailed_tools(original, groups, [*flat[:3], edited, flat[4]]) + + assert list(merged) == [*original[:3], {**_function("dup", "changed"), "strict": False}, original[4]] + assert all(merged[position] is original[position] for position in (0, 1, 2, 4)) + + def test_edited_mcp_tool_is_rewritten(): original = [{"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp"}] groups = _groups(original) From 0346bb265934a09bd5d8eab336facba1cc5bc01b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:58:05 +0000 Subject: [PATCH 109/175] fix(bedrock): upgrade legacy thinking after the invoke response_format stub model swap Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../anthropic_claude3_transformation.py | 5 +++++ ...ations_anthropic_claude3_transformation.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 2a4c38e71ea..07ddf6570f2 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -107,6 +107,11 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): # Restore original model name model = original_model + # The stub model hides the original model from the parent's legacy thinking upgrade + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model=original_model, optional_params=optional_params, custom_llm_provider="bedrock" + ) + # The stub model hides the original model from the parent's forced-tool-use backstop response_format_tool_choice: Final = optional_params.get("tool_choice") if ( diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 41d82e4f960..d136cb6450b 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -671,3 +671,22 @@ def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice assert "output_format" not in result assert "tools" in result assert "tool_choice" not in result + + +def test_bedrock_chat_invoke_response_format_stub_still_upgrades_legacy_thinking(local_model_cost_map): + """Regression: the tool-based ``response_format`` path swaps in a Claude 3 stub + model before the shared Anthropic mapping, which hid the adaptive-only model + from the legacy ``thinking`` upgrade and left ``type=enabled`` on the wire.""" + result = AmazonAnthropicClaudeConfig().map_openai_params( + non_default_params={ + "response_format": {"type": "json_object"}, + "thinking": {"type": "enabled", "budget_tokens": 4096}, + "max_tokens": 8192, + }, + optional_params={}, + model="us.anthropic.claude-fable-5-1", + drop_params=False, + ) + + assert result["thinking"] == {"type": "adaptive"} + assert result["output_config"] == {"effort": "high"} From f4eca10f1d7f832f6300b588434bcd99003f4bb7 Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 20:07:57 +0000 Subject: [PATCH 110/175] ci: retrigger checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> From 0f6d983c7057faf13716639869d828d309bcba5b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:08:14 -0700 Subject: [PATCH 111/175] fix(router): skip Claude Code session binding without pre-routing strategies --- litellm/router.py | 2 ++ tests/test_litellm/test_router.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index fc4f2d227f5..b1038ca6002 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12636,6 +12636,8 @@ class Router: registered_model_name: str, request_kwargs: Mapping[str, object], ) -> str: + if not any((self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers)): + return registered_model_name cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs) if cache_key is None or not isinstance(request_kwargs, dict): return registered_model_name diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 05dcb664ed5..ef25502a4f9 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8499,6 +8499,26 @@ class TestClaudeCodeSubagentSessionRouterBinding: assert response is None redis_cache.async_delete_cache.assert_awaited_once() + @pytest.mark.asyncio + async def test_no_pre_routing_strategies_means_no_session_cache_traffic(self): + from litellm.caching.caching import RedisCache + + router = self._router() + router.complexity_routers = {} + redis_cache = MagicMock(spec=RedisCache) + redis_cache.async_get_cache = AsyncMock(return_value=None) + redis_cache.async_set_cache = AsyncMock() + redis_cache.async_delete_cache = AsyncMock() + router._update_redis_cache(cache=redis_cache) + + for request_kwargs in (self._request_kwargs(), self._request_kwargs(agent_id="agent-1234")): + response = await router.async_pre_routing_hook(model="expensive-model", request_kwargs=request_kwargs) + assert response is None + + redis_cache.async_get_cache.assert_not_awaited() + redis_cache.async_set_cache.assert_not_awaited() + redis_cache.async_delete_cache.assert_not_awaited() + @pytest.mark.asyncio async def test_session_bindings_do_not_evict_router_rate_limit_state(self): router = self._router() From 5da9b7ef900bb60657cd6c4340b3f54833463be2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:12:05 -0700 Subject: [PATCH 112/175] fix(otel): stamp the Langfuse root observation from the post-guardrail request and response --- litellm/integrations/otel/langfuse_logger.py | 43 ++++++--------- litellm/integrations/otel/logger.py | 5 +- .../integrations/otel/test_langfuse_logger.py | 52 +++++++++++++------ 3 files changed, 57 insertions(+), 43 deletions(-) diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py index 9986eae4d0a..ed47533e700 100644 --- a/litellm/integrations/otel/langfuse_logger.py +++ b/litellm/integrations/otel/langfuse_logger.py @@ -8,41 +8,26 @@ from litellm.integrations.otel.model.request_io import request_input, response_o from litellm.integrations.otel.plumbing.context import request_root_span if TYPE_CHECKING: - from litellm.caching.dual_cache import DualCache from litellm.proxy._types import UserAPIKeyAuth - from litellm.types.utils import CallTypesLiteral, ModelResponseStream - -ROOT_OBSERVATION_IO_CALL_TYPES: Final = frozenset( - {"completion", "acompletion", "responses", "aresponses", "anthropic_messages", "aanthropic_messages"} -) + from litellm.types.utils import ModelResponseStream class LangfuseOpenTelemetryV2(OpenTelemetryV2): """Stamps the request's input and output on the root observation while it is still recording. Langfuse shows a trace's input and output from its root observation. The proxy's root span ends - when the response is sent, before the success callback runs, so the stamps have to come from the - request-task hooks: input at pre-call, output at post-call success or at the end of the stream. + when the response is sent, before the success callback runs, so both stamps come from the + post-call hooks in the request task: the request as it stands after the pre-call chain and the + response as it is returned, for the call types whose response renders as a message. """ - async def async_pre_call_hook( - self, - user_api_key_dict: "UserAPIKeyAuth", - cache: "DualCache", - data: Mapping[str, object], - call_type: "CallTypesLiteral", - ) -> None: - await super().async_pre_call_hook(user_api_key_dict, cache, data, call_type) - if call_type in ROOT_OBSERVATION_IO_CALL_TYPES: - self._stamp_root(LANGFUSE_OBSERVATION_INPUT, lambda: request_input(data)) - async def async_post_call_success_hook( self, data: Mapping[str, object], user_api_key_dict: "UserAPIKeyAuth", response: object, ) -> None: - self._stamp_root(LANGFUSE_OBSERVATION_OUTPUT, lambda: response_output(response)) + self._stamp_root_io(data, lambda: response_output(response)) async def async_post_call_streaming_iterator_hook( self, @@ -54,16 +39,22 @@ class LangfuseOpenTelemetryV2(OpenTelemetryV2): async for chunk in response: relayed.append(chunk) yield chunk - self._stamp_root(LANGFUSE_OBSERVATION_OUTPUT, lambda: stream_output(tuple(relayed), request_data)) + self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data)) - def _stamp_root(self, key: str, render: Callable[[], str | None]) -> None: + def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: root: Final = request_root_span() if root is None or not root.is_recording(): return try: - value: Final = render() + output: Final = render_output() + if output is None: + return + root.set_attribute(LANGFUSE_OBSERVATION_OUTPUT, output) + rendered_input: Final = request_input(data) except Exception: # noqa: BLE001 # telemetry must never fail the request it describes - verbose_logger.debug("otel v2 langfuse: could not render %s for the root observation", key, exc_info=True) + verbose_logger.debug( + "otel v2 langfuse: could not render the root observation input or output", exc_info=True + ) return - if value is not None: - root.set_attribute(key, value) + if rendered_input is not None: + root.set_attribute(LANGFUSE_OBSERVATION_INPUT, rendered_input) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 4ab1c738488..a550dca6cc8 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -723,13 +723,14 @@ class OpenTelemetryV2(CustomLogger): self, user_api_key_dict: "UserAPIKeyAuth", cache: "DualCache", - data: Mapping[str, object], + data: dict, call_type: "CallTypesLiteral", - ) -> None: + ) -> dict: self.seed_request_identity( user_api_key_dict, model=model_from_request_data(data), ) + return data def record_error_attributes_on_span( self, diff --git a/tests/test_litellm/integrations/otel/test_langfuse_logger.py b/tests/test_litellm/integrations/otel/test_langfuse_logger.py index 3e35395389e..8f93a9a564f 100644 --- a/tests/test_litellm/integrations/otel/test_langfuse_logger.py +++ b/tests/test_litellm/integrations/otel/test_langfuse_logger.py @@ -31,6 +31,8 @@ from litellm.types.llms.openai import ( # noqa: E402 from litellm.types.utils import ( # noqa: E402 Choices, Delta, + Embedding, + EmbeddingResponse, Message, ModelResponse, ModelResponseStream, @@ -258,26 +260,41 @@ def test_root_observation_io_survives_the_root_ending_before_the_success_callbac assert OUTPUT_ATTR in dict(generation.attributes or {}) +def test_root_input_is_the_request_as_the_pre_call_chain_left_it(): + logger, exporter = _logger() + raw = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "my ssn is 123-45-6789"}]} + masked = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "my ssn is [REDACTED]"}]} + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="noted"))]) + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), raw, "acompletion")) + asyncio.run(logger.async_post_call_success_hook(data=masked, user_api_key_dict=UserAPIKeyAuth(), response=response)) + root.end() + + assert json.loads(_root_attrs(exporter)[INPUT_ATTR]) == masked["messages"] + + def test_root_already_ended_is_left_alone(): logger, exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) root = _start_root(logger) root.end() - asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) - - assert INPUT_ATTR not in _root_attrs(exporter) - - -def test_non_chat_call_types_do_not_stamp_input(): - logger, exporter = _logger() - root = _start_root(logger) - asyncio.run( - logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), {"model": "e", "input": "ping"}, "aembedding") + logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) ) - root.end() - assert INPUT_ATTR not in _root_attrs(exporter) + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs + + +def test_responses_without_a_message_body_stamp_neither_input_nor_output(): + logger, exporter = _logger() + embedding = EmbeddingResponse(model="e", data=[Embedding(embedding=[0.1], index=0, object="embedding")]) + + _run_request(logger, {"model": "e", "input": "ping"}, "aembedding", embedding) + + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs def test_unrenderable_output_never_raises_into_the_request(): @@ -285,7 +302,8 @@ def test_unrenderable_output_never_raises_into_the_request(): _run_request(logger, CHAT_DATA, "acompletion", object()) - assert OUTPUT_ATTR not in _root_attrs(exporter) + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs @pytest.mark.parametrize( @@ -331,7 +349,11 @@ def test_langfuse_otel_preset_builds_a_logger_that_stamps_the_root(monkeypatch): assert built is not None assert _maybe_construct_otel_v2("langfuse_otel", loggers) is built root = _start_root(built) - asyncio.run(built.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) - assert INPUT_ATTR in dict(root.attributes or {}) + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + asyncio.run( + built.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + attrs = dict(root.attributes or {}) + assert INPUT_ATTR in attrs and OUTPUT_ATTR in attrs finally: is_otel_v2_enabled.cache_clear() From 6fae4b3c3977edcddfeb9e0080f91718ae5c77d6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:32:21 -0700 Subject: [PATCH 113/175] fix(guardrails): keep the presidio output masker from unmasking after an in-memory update --- .../proxy/guardrails/guardrail_hooks/presidio.py | 2 ++ .../guardrails/guardrail_hooks/test_presidio.py | 12 ++++++++++++ .../proxy/guardrails/test_guardrail_registry.py | 13 +++++++++++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index da51a905ae3..70ea21320ee 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -1633,6 +1633,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): Update the guardrails litellm params in memory """ super().update_in_memory_litellm_params(litellm_params) + if self.apply_to_output: + self.output_parse_pii = False if litellm_params.pii_entities_config: self.pii_entities_config = litellm_params.pii_entities_config if litellm_params.presidio_score_thresholds: diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index fcf940afd0d..84f7611c0c0 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -3129,6 +3129,18 @@ def test_update_in_memory_applies_analyze_chunk_size(): assert guardrail.presidio_analyze_chunk_size_bytes == 99_000 +def test_update_in_memory_keeps_output_masker_from_unmasking(): + masker = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True, output_parse_pii=False) + unmasker = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True) + params = LitellmParams(guardrail="presidio", mode="pre_call", output_parse_pii=True) + + masker.update_in_memory_litellm_params(params) + unmasker.update_in_memory_litellm_params(params) + + assert (masker.apply_to_output, masker.output_parse_pii) == (True, False) + assert (unmasker.apply_to_output, unmasker.output_parse_pii) == (False, True) + + def test_merge_drops_truncated_same_type_fragment_from_overlap(): """A boundary entity seen truncated by chunk 1 and whole by chunk 2 must merge to the single full span; keeping both overlapping spans corrupts the diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 5cbdef5f92f..24742e1bac2 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -566,7 +566,14 @@ def test_update_in_memory_guardrail_reaches_presidio_siblings_and_keeps_their_st try: handler.initialize_guardrail(_presidio_db_guardrail({"EMAIL_ADDRESS": "MASK", "IP_ADDRESS": "MASK"})) tracked = _presidio_callbacks_in(litellm.callbacks) - roles_before = [(callback.apply_to_output, callback.event_hook) for callback in tracked] + roles_before = [ + (callback.apply_to_output, callback.output_parse_pii, callback.event_hook) for callback in tracked + ] + assert roles_before == [ + (False, True, [GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call]), + (False, True, GuardrailEventHooks.post_call), + (True, False, GuardrailEventHooks.post_call), + ] updated = Guardrail( guardrail_id=PRESIDIO_SIBLINGS_GID, @@ -585,7 +592,9 @@ def test_update_in_memory_guardrail_reaches_presidio_siblings_and_keeps_their_st handler.update_in_memory_guardrail(guardrail_id=PRESIDIO_SIBLINGS_GID, guardrail=updated) assert [callback.pii_entities_config for callback in tracked] == [{"EMAIL_ADDRESS": "MASK"}] * 3 - assert [(callback.apply_to_output, callback.event_hook) for callback in tracked] == roles_before + assert [ + (callback.apply_to_output, callback.output_parse_pii, callback.event_hook) for callback in tracked + ] == roles_before assert _presidio_callbacks_in(litellm.callbacks) == tracked finally: for cb_list, snapshot in zip(lists, snapshots): From 711430216ea73eb0ad45773a81c61155062e0567 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 2 Sep 2026 13:33:14 -0700 Subject: [PATCH 114/175] fix(ui): preserve full AgentCore runtime ARN in agent edit form (#39382) parseDynamicAgentForForm recovered a credential field's value from a stored model string by splitting both the model_template and the model on "/" and matching by array index. That breaks for any placeholder value that itself contains "/", such as a Bedrock AgentCore runtime ARN resource path (runtime/), silently dropping everything after the first slash when populating the edit form. Saving without touching the field then persisted the truncated ARN. Replace the index-matching split with a non-mutating template parse (split on the placeholder pattern, escape and rejoin the literal segments into a regex) so a placeholder captures everything it needs regardless of embedded slashes. Also add a lightweight ARN-shape validator for the AgentCore runtime ARN field, guarded against a malformed pattern string, so a truncated value is rejected client-side before it reaches the backend. Resolves LIT-6737 --- .../public_endpoints/agent_create_fields.json | 4 +- .../public_endpoints/public_endpoints.py | 2 + .../public_endpoints/test_public_endpoints.py | 40 ++++++ .../agent_info.integration.test.tsx | 97 ++++++++++++++ .../_components/agent_type_utils.test.ts | 73 ++++++++++ .../agents/_components/agent_type_utils.ts | 45 +++++-- .../_components/dynamic_agent_form_fields.tsx | 126 ++++++++++-------- .../src/components/networking.tsx | 2 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 9 files changed, 326 insertions(+), 67 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.test.ts diff --git a/litellm/proxy/public_endpoints/agent_create_fields.json b/litellm/proxy/public_endpoints/agent_create_fields.json index 36484cc1065..cc2fc17d759 100644 --- a/litellm/proxy/public_endpoints/agent_create_fields.json +++ b/litellm/proxy/public_endpoints/agent_create_fields.json @@ -107,7 +107,9 @@ "required": true, "field_type": "text", "default_value": null, - "include_in_litellm_params": false + "include_in_litellm_params": false, + "validation_pattern": "^arn:aws[a-zA-Z0-9-]*:bedrock-agentcore:[a-z0-9-]+:[0-9]{12}:runtime/.+$", + "validation_message": "Enter the complete Bedrock AgentCore runtime ARN, including the runtime ID after \"runtime/\" (e.g. arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime)." } ], "litellm_params_template": { diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index f6ee054ceaa..c7f80a61e0f 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -43,6 +43,8 @@ class AgentCredentialField(BaseModel): options: list[str] | None = None default_value: str | None = None include_in_litellm_params: bool | None = None + validation_pattern: str | None = None + validation_message: str | None = None class AgentCreateInfo(BaseModel): diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 31430da71e8..8006f64ba41 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -1,3 +1,4 @@ +import re from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch @@ -756,6 +757,45 @@ def test_public_agent_hub_returns_empty_when_no_public_groups(): assert response.json() == [] +# --------------------------------------------------------------------------- +# /public/agents/fields +# --------------------------------------------------------------------------- + + +def test_bedrock_agentcore_runtime_arn_validation_pattern_accepts_full_resource_path(): + """Regression for LIT-6737: the AgentCore agent_runtime_arn field's + validation_pattern must accept a complete runtime ARN whose resource part + is itself multi-segment (``runtime/``), and reject the exact + truncated shape a naive split("/")-by-position parse used to produce (the + ARN cut off right after the ``runtime`` resource type). + """ + app_instance = FastAPI() + app_instance.include_router(router) + test_client = TestClient(app_instance) + + response = test_client.get("/public/agents/fields") + assert response.status_code == 200 + agents = response.json() + + bedrock_agentcore = next((a for a in agents if a["agent_type"] == "bedrock_agentcore"), None) + assert bedrock_agentcore is not None, "bedrock_agentcore agent type not found" + assert bedrock_agentcore["model_template"] == "bedrock/agentcore/{agent_runtime_arn}" + + fields_by_key = {f["key"]: f for f in bedrock_agentcore["credential_fields"]} + arn_field = fields_by_key["agent_runtime_arn"] + assert arn_field["required"] is True + assert arn_field["include_in_litellm_params"] is False + + pattern = arn_field.get("validation_pattern") + assert pattern, "agent_runtime_arn must ship a validation_pattern so the UI can reject a truncated ARN" + + full_arn = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime/hosted_agent_4vm3i-BaTdfOELAs" + truncated_arn = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime" + + assert re.match(pattern, full_arn), "the validator must accept a complete runtime ARN" + assert not re.match(pattern, truncated_arn), "the validator must reject the truncated ARN" + + # --------------------------------------------------------------------------- # /public/endpoints # --------------------------------------------------------------------------- diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx index 79bd2f6a21b..de1eb153f6f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx @@ -75,6 +75,40 @@ const langgraphInfo: AgentCreateInfo = { ], }; +const FULL_RUNTIME_ARN = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime/hosted_agent_4vm3i-BaTdfOELAs"; + +const BEDROCK_AGENTCORE_AGENT = { + agent_id: "agent-3", + agent_name: "bedrock-agent", + agent_card_params: { name: "bedrock-agent", description: "agentcore agent", url: "", version: "1.0.0", skills: [] }, + litellm_params: { + custom_llm_provider: "bedrock", + model: `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + }, +}; + +const bedrockAgentcoreInfo: AgentCreateInfo = { + agent_type: "bedrock_agentcore", + agent_type_display_name: "Bedrock AgentCore", + description: "Bedrock AgentCore runtimes", + logo_url: "/b.png", + use_a2a_form_fields: false, + litellm_params_template: { custom_llm_provider: "bedrock" }, + model_template: "bedrock/agentcore/{agent_runtime_arn}", + credential_fields: [ + { + key: "agent_runtime_arn", + label: "Agent Runtime ARN", + field_type: "text", + required: true, + include_in_litellm_params: false, + validation_pattern: "^arn:aws[a-zA-Z0-9-]*:bedrock-agentcore:[a-z0-9-]+:[0-9]{12}:runtime/.+$", + validation_message: + 'Enter the complete Bedrock AgentCore runtime ARN, including the runtime ID after "runtime/".', + }, + ], +}; + const setup = () => userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); const renderView = () => render(); @@ -247,6 +281,69 @@ describe("AgentInfoView update payload", () => { }); }); + it("preserves the full AgentCore runtime ARN (including the resource id after runtime/) across an unedited save", async () => { + vi.mocked(networking.getAgentCreateMetadata).mockResolvedValue([bedrockAgentcoreInfo]); + vi.mocked(networking.getAgentInfo).mockResolvedValue(BEDROCK_AGENTCORE_AGENT as never); + const user = setup(); + renderView(); + await openEditor(user); + + expect(await screen.findByLabelText("Agent Runtime ARN")).toHaveValue(FULL_RUNTIME_ARN); + + await save(user); + + expect((patchedPayload().litellm_params as Record).model).toBe( + `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + ); + }); + + it("blocks the save and shows a validation error when the Agent Runtime ARN is truncated", async () => { + vi.mocked(networking.getAgentCreateMetadata).mockResolvedValue([bedrockAgentcoreInfo]); + vi.mocked(networking.getAgentInfo).mockResolvedValue(BEDROCK_AGENTCORE_AGENT as never); + const user = setup(); + renderView(); + await openEditor(user); + + const arnField = await screen.findByLabelText("Agent Runtime ARN"); + await user.clear(arnField); + fireEvent.change(arnField, { + target: { value: "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime" }, + }); + await user.click(screen.getByRole("button", { name: "Save Changes" })); + + expect( + await screen.findByText( + 'Enter the complete Bedrock AgentCore runtime ARN, including the runtime ID after "runtime/".', + ), + ).toBeInTheDocument(); + expect(networking.patchAgentCall).not.toHaveBeenCalled(); + }); + + it("renders and saves normally when a field's validation_pattern is not a valid regex", async () => { + const infoWithBadPattern: AgentCreateInfo = { + ...bedrockAgentcoreInfo, + credential_fields: [ + { + ...bedrockAgentcoreInfo.credential_fields[0], + validation_pattern: "(unterminated", + }, + ], + }; + vi.mocked(networking.getAgentCreateMetadata).mockResolvedValue([infoWithBadPattern]); + vi.mocked(networking.getAgentInfo).mockResolvedValue(BEDROCK_AGENTCORE_AGENT as never); + const user = setup(); + renderView(); + await openEditor(user); + + expect(await screen.findByLabelText("Agent Runtime ARN")).toHaveValue(FULL_RUNTIME_ARN); + + await save(user); + + expect((patchedPayload().litellm_params as Record).model).toBe( + `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + ); + }); + it("reloads the agent and leaves edit mode when the edit is cancelled", async () => { const user = setup(); renderView(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.test.ts new file mode 100644 index 00000000000..0c2d2500776 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.test.ts @@ -0,0 +1,73 @@ +import { describe, it, expect } from "vitest"; +import { detectAgentType, extractModelTemplateValues, parseDynamicAgentForForm } from "./agent_type_utils"; +import type { AgentCreateInfo } from "@/components/networking"; +import type { Agent } from "@/components/agents/types"; + +const FULL_RUNTIME_ARN = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime/hosted_agent_4vm3i-BaTdfOELAs"; + +const bedrockAgentcoreInfo: AgentCreateInfo = { + agent_type: "bedrock_agentcore", + agent_type_display_name: "Bedrock AgentCore", + model_template: "bedrock/agentcore/{agent_runtime_arn}", + credential_fields: [ + { + key: "agent_runtime_arn", + label: "Agent Runtime ARN", + required: true, + include_in_litellm_params: false, + }, + ], +}; + +describe("extractModelTemplateValues", () => { + it("recovers a placeholder value that itself contains '/' (an AWS ARN resource path)", () => { + const values = extractModelTemplateValues( + "bedrock/agentcore/{agent_runtime_arn}", + `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + ); + + expect(values.agent_runtime_arn).toBe(FULL_RUNTIME_ARN); + }); + + it("recovers a placeholder value with no '/' (single path segment)", () => { + const values = extractModelTemplateValues("langgraph/{assistant_id}", "langgraph/asst_1"); + + expect(values.assistant_id).toBe("asst_1"); + }); + + it("returns no match when the model does not fit the template", () => { + const values = extractModelTemplateValues("langgraph/{assistant_id}", "azure_ai/agents/asst_1"); + + expect(values).toEqual({}); + }); +}); + +describe("parseDynamicAgentForForm", () => { + it("preserves the full runtime ARN, including the resource id after 'runtime/', when populating the edit form", () => { + const agent = { + agent_id: "agent-1", + agent_name: "bedrock-agent", + agent_card_params: { description: "" }, + litellm_params: { + custom_llm_provider: "bedrock", + model: `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + }, + } as unknown as Agent; + + const values = parseDynamicAgentForForm(agent, bedrockAgentcoreInfo); + + expect(values.agent_runtime_arn).toBe(FULL_RUNTIME_ARN); + }); +}); + +describe("detectAgentType", () => { + it("detects bedrock_agentcore agents from the model prefix", () => { + const agent = { + agent_id: "agent-1", + agent_name: "bedrock-agent", + litellm_params: { model: `bedrock/agentcore/${FULL_RUNTIME_ARN}` }, + } as unknown as Agent; + + expect(detectAgentType(agent)).toBe("bedrock_agentcore"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts index f91590c5732..506f355c519 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts @@ -25,6 +25,29 @@ export const detectAgentType = (agent: Agent): string => { return "a2a"; }; +const escapeRegExp = (segment: string): string => segment.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +/** + * Reverses a `model_template` (e.g. "bedrock/agentcore/{agent_runtime_arn}") against a stored + * `model` string to recover the placeholder values that produced it. Builds a regex from the + * template's literal segments rather than matching by split("/") position, because a + * placeholder's value can itself contain "/" (an AWS ARN's "runtime/" resource path, + * a Vertex AI reasoning engine's "projects/.../reasoningEngines/..." resource id) and would + * otherwise be cut off at the first one. + */ +export const extractModelTemplateValues = (template: string, model: string): Record => { + // Splitting on a regex with a capturing group interleaves the captured placeholder + // names between the surrounding literal segments, e.g. "a/{x}/b" -> ["a/", "x", "/b"]. + const parts = template.split(/\{([a-zA-Z0-9_]+)\}/g); + const fieldNames = parts.filter((_part, index) => index % 2 === 1); + const pattern = parts.map((part, index) => (index % 2 === 1 ? "(.+)" : escapeRegExp(part))).join(""); + + const match = model.match(new RegExp(`^${pattern}$`)); + if (!match) return {}; + + return Object.fromEntries(fieldNames.map((name, index) => [name, match[index + 1]])); +}; + /** * Parses agent data for dynamic form fields (non-A2A agents). * Extracts values from litellm_params based on the agent type metadata. @@ -35,24 +58,18 @@ export const parseDynamicAgentForForm = (agent: Agent, agentTypeInfo: AgentCreat description: agent.agent_card_params?.description || "", }; + const templateValues = + agentTypeInfo.model_template && agent.litellm_params?.model + ? extractModelTemplateValues(agentTypeInfo.model_template, agent.litellm_params.model) + : {}; + // Extract credential field values from litellm_params for (const field of agentTypeInfo.credential_fields) { if (field.include_in_litellm_params !== false) { values[field.key] = agent.litellm_params?.[field.key] || field.default_value || ""; - } else { - // For fields not in litellm_params (like agent_id), try to extract from model string - if (agentTypeInfo.model_template && agent.litellm_params?.model) { - const model = agent.litellm_params.model; - const templateParts = agentTypeInfo.model_template.split("/"); - const modelParts = model.split("/"); - - // Find the placeholder position and extract the value - templateParts.forEach((part, index) => { - if (part === `{${field.key}}` && modelParts[index]) { - values[field.key] = modelParts[index]; - } - }); - } + } else if (templateValues[field.key] !== undefined) { + // For fields not in litellm_params (like agent_runtime_arn), recover from the model string + values[field.key] = templateValues[field.key]; } } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx index 04a8b0df9d9..0f3355fe073 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx @@ -24,58 +24,80 @@ interface DynamicAgentFormFieldsProps { export const unmountedDynamicFieldNames = (mountedPanels: readonly string[]): readonly string[] => mountedPanels.includes(AGENT_FORM_CONFIG.cost.key) ? [] : COST_FIELD_NAMES; -const CredentialField = ({ field }: { field: AgentCredentialFieldMetadata }) => ( - - {({ value, onChange, ref, ...control }) => { - const text = typeof value === "string" ? value : ""; - if (field.field_type === "password") { - return ( - - ); - } - if (field.field_type === "textarea") { - return ( -