From a1514efa210c60c00809b21d2906503b0c452cc8 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Mon, 27 Jul 2026 17:07:58 +0200 Subject: [PATCH 01/67] fix(vector_stores): S3 Vectors search router bypass + rag query config drop + UI error swallow --- .../azure_ai/vector_stores/transformation.py | 2 + .../base_llm/vector_store/transformation.py | 4 + .../bedrock/vector_stores/transformation.py | 2 + litellm/llms/custom_httpx/llm_http_handler.py | 7 + .../gemini/vector_stores/transformation.py | 2 + .../milvus/vector_stores/transformation.py | 2 + .../openai/vector_stores/transformation.py | 2 + .../pg_vector/vector_stores/transformation.py | 2 + .../ragflow/vector_stores/transformation.py | 2 + .../vector_stores/transformation.py | 32 ++- .../vector_stores/rag_api/transformation.py | 2 + .../search_api/transformation.py | 2 + litellm/proxy/rag_endpoints/endpoints.py | 14 ++ litellm/rag/main.py | 14 +- litellm/router.py | 8 + litellm/vector_stores/main.py | 9 +- .../test_s3_vectors_transformation.py | 189 +++++++++++++++++- .../proxy/rag_endpoints/test_rag_endpoints.py | 103 ++++++++++ tests/test_litellm/rag/test_main.py | 90 +++++++++ tests/test_litellm/test_router.py | 55 +++++ tests/test_litellm/vector_stores/test_main.py | 77 +++++++ .../_components/VectorStoreTester.test.tsx | 25 ++- .../_components/VectorStoreTester.tsx | 8 +- .../src/components/networking.tsx | 2 +- 24 files changed, 628 insertions(+), 27 deletions(-) create mode 100644 tests/test_litellm/vector_stores/test_main.py diff --git a/litellm/llms/azure_ai/vector_stores/transformation.py b/litellm/llms/azure_ai/vector_stores/transformation.py index da6a4a93cd8..bd3eaeee989 100644 --- a/litellm/llms/azure_ai/vector_stores/transformation.py +++ b/litellm/llms/azure_ai/vector_stores/transformation.py @@ -19,6 +19,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -92,6 +93,7 @@ class AzureAIVectorStoreConfig(BaseVectorStoreConfig, BaseAzureLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Azure AI Search API diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index b222e3dd160..9a0e401b527 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -16,6 +16,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router from ..chat.transformation import BaseLLMException as _BaseLLMException @@ -56,6 +57,7 @@ class BaseVectorStoreConfig: litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: pass @@ -68,6 +70,7 @@ class BaseVectorStoreConfig: litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """ Optional async version of transform_search_vector_store_request. @@ -83,6 +86,7 @@ class BaseVectorStoreConfig: litellm_logging_obj=litellm_logging_obj, litellm_params=litellm_params, extra_body=extra_body, + router=router, ) @abstractmethod diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index c1b124caec1..7a6a0eb6d84 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -27,6 +27,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -196,6 +197,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: if isinstance(query, list): query = " ".join(query) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ec1301e5923..ec701fbe87e 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -167,6 +167,7 @@ if TYPE_CHECKING: AnthropicMessagesStreamingResponse, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig + from litellm.router import Router from litellm.types.llms.openai_evals import ( CancelEvalResponse, CancelRunResponse, @@ -9409,6 +9410,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + router: Optional["Router"] = None, ) -> VectorStoreSearchResponse: if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( @@ -9443,6 +9445,7 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, + router=router, ) else: ( @@ -9456,6 +9459,7 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, + router=router, ) all_optional_params: Dict[str, Any] = dict(litellm_params) all_optional_params.update(vector_store_search_optional_params or {}) @@ -9507,6 +9511,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + router: Optional["Router"] = None, ) -> Union[VectorStoreSearchResponse, Coroutine[Any, Any, VectorStoreSearchResponse]]: if _is_async: return self.async_vector_store_search_handler( @@ -9521,6 +9526,7 @@ class BaseLLMHTTPHandler: extra_body=extra_body, timeout=timeout, client=client, + router=router, ) if client is None or not isinstance(client, HTTPHandler): @@ -9551,6 +9557,7 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, + router=router, ) all_optional_params: Dict[str, Any] = dict(litellm_params) diff --git a/litellm/llms/gemini/vector_stores/transformation.py b/litellm/llms/gemini/vector_stores/transformation.py index f98cb0e5b0c..5aba5752a44 100644 --- a/litellm/llms/gemini/vector_stores/transformation.py +++ b/litellm/llms/gemini/vector_stores/transformation.py @@ -31,6 +31,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -111,6 +112,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """ Transform search request to Gemini's generateContent format. diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py index a53075ba1d6..589063cd188 100644 --- a/litellm/llms/milvus/vector_stores/transformation.py +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -19,6 +19,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -123,6 +124,7 @@ class MilvusVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Azure AI Search API diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py index 6ccf8e271e5..9ab1568a375 100644 --- a/litellm/llms/openai/vector_stores/transformation.py +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -21,6 +21,7 @@ from litellm.utils import add_openai_metadata if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -99,6 +100,7 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: encoded_vector_store_id = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url = f"{api_base}/{encoded_vector_store_id}/search" diff --git a/litellm/llms/pg_vector/vector_stores/transformation.py b/litellm/llms/pg_vector/vector_stores/transformation.py index b58b6e7f498..116f79c834f 100644 --- a/litellm/llms/pg_vector/vector_stores/transformation.py +++ b/litellm/llms/pg_vector/vector_stores/transformation.py @@ -8,6 +8,7 @@ from litellm.types.vector_stores import VectorStoreSearchOptionalRequestParams if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -80,6 +81,7 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: encoded_vector_store_id = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url = f"{api_base}/{encoded_vector_store_id}/search" diff --git a/litellm/llms/ragflow/vector_stores/transformation.py b/litellm/llms/ragflow/vector_stores/transformation.py index d8bdd981425..332ed7f0c6b 100644 --- a/litellm/llms/ragflow/vector_stores/transformation.py +++ b/litellm/llms/ragflow/vector_stores/transformation.py @@ -17,6 +17,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -92,6 +93,7 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """RAGFlow vector stores are management-only, search is not supported.""" raise NotImplementedError("RAGFlow vector stores support dataset management only, not search/retrieval") diff --git a/litellm/llms/s3_vectors/vector_stores/transformation.py b/litellm/llms/s3_vectors/vector_stores/transformation.py index b31e6f4511a..a999db21dbe 100644 --- a/litellm/llms/s3_vectors/vector_stores/transformation.py +++ b/litellm/llms/s3_vectors/vector_stores/transformation.py @@ -1,8 +1,8 @@ -import re from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import httpx +from litellm.caching._embedding_router import resolve_embedding_router from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.types.router import GenericLiteLLMParams @@ -18,6 +18,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -58,13 +59,18 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): return headers def get_complete_url(self, api_base: Optional[str], litellm_params: dict) -> str: - aws_region_name = litellm_params.get("aws_region_name") - if not aws_region_name: - raise ValueError("aws_region_name is required for S3 Vectors") - if not re.match(r"^[a-z][a-z0-9-]*$", aws_region_name): - raise ValueError("Invalid aws_region_name format") + # Resolve region the same way the ingestion path does: + # dynamic param -> AWS_REGION_NAME -> AWS_REGION -> default (us-west-2) + aws_region_name = self.get_aws_region_name_for_non_llm_api_calls(litellm_params.get("aws_region_name")) return f"https://s3vectors.{aws_region_name}.api.aws" + def _resolve_query_embedding_router(self, embedding_model: str, router: Optional["Router"]) -> Optional["Router"]: + """Return the router iff it serves ``embedding_model`` as a deployment.""" + if router is None: + return None + model_list = [dict(m) for m in (router.get_model_list() or [])] + return resolve_embedding_router(embedding_model=embedding_model, llm_router=router, llm_model_list=model_list) + def transform_search_vector_store_request( self, vector_store_id: str, @@ -74,6 +80,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """Sync version - generates embedding synchronously.""" # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name @@ -99,10 +106,14 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): # Generate embedding for the query embedding_model = litellm_params.get("embedding_model", "text-embedding-3-small") + embedding_router = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router) import litellm as litellm_module - embedding_response = litellm_module.embedding(model=embedding_model, input=[query]) + if embedding_router is not None: + embedding_response = embedding_router.embedding(model=embedding_model, input=[query]) + else: + embedding_response = litellm_module.embedding(model=embedding_model, input=[query]) query_embedding = embedding_response.data[0]["embedding"] url = f"{api_base}/QueryVectors" @@ -128,6 +139,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """Async version - generates embedding asynchronously.""" # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name @@ -153,10 +165,14 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): # Generate embedding for the query asynchronously embedding_model = litellm_params.get("embedding_model", "text-embedding-3-small") + embedding_router = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router) import litellm as litellm_module - embedding_response = await litellm_module.aembedding(model=embedding_model, input=[query]) + if embedding_router is not None: + embedding_response = await embedding_router.aembedding(model=embedding_model, input=[query]) + else: + embedding_response = await litellm_module.aembedding(model=embedding_model, input=[query]) query_embedding = embedding_response.data[0]["embedding"] url = f"{api_base}/QueryVectors" diff --git a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py index 47a81fc07bf..93ad40616b5 100644 --- a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py @@ -19,6 +19,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -97,6 +98,7 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Vertex AI RAG API diff --git a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py index 958839d4a48..f6f9e34dc75 100644 --- a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py @@ -23,6 +23,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -197,6 +198,7 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform a search request for the Vertex AI Search (Discovery Engine) API. diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index 27ffc49901b..0d93f20373c 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -26,6 +26,9 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _safe_get_request_headers, get_form_data, ) +from litellm.proxy.vector_store_endpoints.endpoints import ( + _update_request_data_with_litellm_managed_vector_store_registry, +) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, ) @@ -652,6 +655,17 @@ async def rag_query( user_api_key_dict=user_api_key_dict, ) + # Merge litellm-managed vector store params (provider, region, embedding + # model, credentials, ...) from the registry — same source the direct + # /vector_stores/{id}/search endpoint uses. User-supplied + # retrieval_config keys win on conflict. + store_data = await _update_request_data_with_litellm_managed_vector_store_registry( + data={}, + vector_store_id=retrieval_config["vector_store_id"], + user_api_key_dict=user_api_key_dict, + ) + retrieval_config = {**store_data, **retrieval_config} + # Add litellm data request_data: Dict[str, Any] = {} request_data = await add_litellm_data_to_request( diff --git a/litellm/rag/main.py b/litellm/rag/main.py index 29891ccfd24..2329a820f1f 100644 --- a/litellm/rag/main.py +++ b/litellm/rag/main.py @@ -59,6 +59,14 @@ INGESTION_REGISTRY: Dict[str, Type[BaseRAGIngestion]] = { "vertex_ai": VertexAIRAGIngestion, } +# retrieval_config keys consumed by the query pipeline itself; everything else is +# forwarded to vector_stores.asearch as provider-specific params (e.g. +# aws_region_name, embedding_model, vector_bucket_name for S3 Vectors). +# `filters`/`retrieval_filter` are reserved for the explicit filter param. +_CONSUMED_RETRIEVAL_CONFIG_KEYS = frozenset( + {"vector_store_id", "custom_llm_provider", "top_k", "filters", "retrieval_filter"} +) + def get_ingestion_class(provider: str) -> Type[BaseRAGIngestion]: """ @@ -233,13 +241,17 @@ async def _execute_query_pipeline( raise ValueError("No query found in messages for RAG query") # 2. Search vector store + # Forward provider-specific retrieval_config extras (region, embedding model, + # bucket, credentials refs, ...) to the search call; kwargs win on conflict. + provider_search_params = {k: v for k, v in retrieval_config.items() if k not in _CONSUMED_RETRIEVAL_CONFIG_KEYS} with _suppressed_sub_call_billing(): search_response = await litellm.vector_stores.asearch( vector_store_id=retrieval_config["vector_store_id"], query=query_text, max_num_results=retrieval_config.get("top_k", 10), custom_llm_provider=retrieval_config.get("custom_llm_provider", "openai"), - **kwargs, + router=router, + **{**provider_search_params, **kwargs}, ) search_provider = retrieval_config.get("custom_llm_provider", "openai") diff --git a/litellm/router.py b/litellm/router.py index 78fe3ff025e..bffd1df3814 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5820,6 +5820,7 @@ class Router: return await self._init_vector_store_api_endpoints( original_function=original_function, custom_llm_provider=custom_llm_provider, + call_type=call_type, **kwargs, ) elif call_type in ("afile_delete", "afile_content"): @@ -5860,6 +5861,7 @@ class Router: self, original_function: Callable, custom_llm_provider: Optional[str] = None, + call_type: Optional[str] = None, **kwargs, ): """ @@ -5878,6 +5880,12 @@ class Router: **kwargs, ) + # For search, pass the router so provider transforms can resolve + # router-managed embedding models (e.g. S3 Vectors query embeddings). + # Assigning into kwargs also overrides any client-supplied `router` key. + if call_type == "avector_store_search": + kwargs["router"] = self + # Otherwise, call the original function directly return await original_function(**kwargs) diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index f768ee75545..4035125120e 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -6,7 +6,7 @@ import asyncio import builtins import contextvars from functools import partial -from typing import Any, Coroutine, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Union import httpx @@ -28,6 +28,9 @@ from litellm.types.vector_stores import ( from litellm.utils import ProviderConfigManager, client from litellm.vector_stores.utils import VectorStoreRequestUtils +if TYPE_CHECKING: + from litellm.router import Router + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -279,6 +282,7 @@ async def asearch( timeout: Optional[Union[float, httpx.Timeout]] = None, # LiteLLM specific params, custom_llm_provider: Optional[str] = None, + router: Optional["Router"] = None, **kwargs, ) -> VectorStoreSearchResponse: """ @@ -307,6 +311,7 @@ async def asearch( extra_body=extra_body, timeout=timeout, custom_llm_provider=custom_llm_provider, + router=router, **kwargs, ) @@ -346,6 +351,7 @@ def search( timeout: Optional[Union[float, httpx.Timeout]] = None, # LiteLLM specific params, custom_llm_provider: Optional[str] = None, + router: Optional["Router"] = None, **kwargs, ) -> Union[VectorStoreSearchResponse, Coroutine[Any, Any, VectorStoreSearchResponse]]: """ @@ -449,6 +455,7 @@ def search( timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), + router=router, ) return response diff --git a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py index 7085e45cdc3..9389476bef4 100644 --- a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py +++ b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, Mock +from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest @@ -9,6 +9,18 @@ from litellm.llms.s3_vectors.vector_stores.transformation import ( from litellm.types.vector_stores import VectorStoreSearchResponse +def _mock_router(model_names, sync=False): + """Router mock serving the given embedding model names.""" + router = MagicMock() + router.get_model_list.return_value = [{"model_name": name} for name in model_names] + embedding_response = Mock(data=[{"embedding": [0.1, 0.2, 0.3]}]) + if sync: + router.embedding = MagicMock(return_value=embedding_response) + else: + router.aembedding = AsyncMock(return_value=embedding_response) + return router + + class TestS3VectorsVectorStoreConfig: def test_init(self): """Test that S3VectorsVectorStoreConfig initializes correctly""" @@ -28,19 +40,174 @@ class TestS3VectorsVectorStoreConfig: url = config.get_complete_url(None, litellm_params) assert url == "https://s3vectors.us-west-2.api.aws" - def test_get_complete_url_missing_region(self): - """Test that missing region raises error""" + def test_get_complete_url_missing_region(self, monkeypatch): + """Missing region falls back to the default region (parity with ingestion)""" + monkeypatch.delenv("AWS_REGION_NAME", raising=False) + monkeypatch.delenv("AWS_REGION", raising=False) config = S3VectorsVectorStoreConfig() - litellm_params = {} - with pytest.raises(ValueError, match="aws_region_name is required"): - config.get_complete_url(None, litellm_params) + url = config.get_complete_url(None, {}) + assert url == "https://s3vectors.us-west-2.api.aws" + + def test_get_complete_url_uses_env_region(self, monkeypatch): + """Missing region param resolves from AWS_REGION_NAME env var""" + monkeypatch.setenv("AWS_REGION_NAME", "eu-west-1") + monkeypatch.delenv("AWS_REGION", raising=False) + config = S3VectorsVectorStoreConfig() + url = config.get_complete_url(None, {}) + assert url == "https://s3vectors.eu-west-1.api.aws" + + def test_get_complete_url_invalid_region_format(self): + """Invalid region format raises""" + config = S3VectorsVectorStoreConfig() + with pytest.raises(ValueError, match="Invalid AWS region format"): + config.get_complete_url(None, {"aws_region_name": "Bad_Region!"}) - @pytest.mark.skip(reason="Requires embedding API call, tested in integration tests") def test_transform_search_request(self): - """Test search request transformation""" - # This test requires making an actual embedding API call - # It's better tested in integration tests - pass + """Full request-body transformation with a router-injected embedding""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["text-embedding-3-small"], sync=True) + + url, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={"max_num_results": 7}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + router=router, + ) + + assert url == "https://s3vectors.us-west-2.api.aws/QueryVectors" + assert request_body == { + "vectorBucketName": "test-bucket", + "indexName": "test-index", + "queryVector": {"float32": [0.1, 0.2, 0.3]}, + "topK": 7, + "returnDistance": True, + "returnMetadata": True, + } + assert mock_logging_obj.model_call_details["query"] == "test query" + + @pytest.mark.asyncio + async def test_atransform_search_uses_router_for_virtual_model(self): + """Regression: router-served embedding models must resolve via the router, + not a bare litellm.aembedding call (which has no deployment credentials).""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["my-embedding-model"]) + + with patch("litellm.aembedding", new=AsyncMock()) as mock_bare_aembedding: + url, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "my-embedding-model"}, + extra_body=None, + router=router, + ) + + router.aembedding.assert_awaited_once_with(model="my-embedding-model", input=["test query"]) + mock_bare_aembedding.assert_not_awaited() + assert request_body["queryVector"]["float32"] == [0.1, 0.2, 0.3] + assert request_body["topK"] == 5 # default + + @pytest.mark.asyncio + async def test_atransform_search_falls_back_when_router_does_not_serve_model(self): + """Router present but embedding_model is not a router deployment -> + bare litellm.aembedding keeps working (provider-prefixed + env creds stores).""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["some-other-model"]) + + mock_bare = AsyncMock(return_value=Mock(data=[{"embedding": [0.4, 0.5]}])) + with patch("litellm.aembedding", new=mock_bare): + _, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "azure/text-embedding-3-small"}, + extra_body=None, + router=router, + ) + + mock_bare.assert_awaited_once_with(model="azure/text-embedding-3-small", input=["test query"]) + router.aembedding.assert_not_awaited() + assert request_body["queryVector"]["float32"] == [0.4, 0.5] + + @pytest.mark.asyncio + async def test_atransform_search_without_router_uses_bare_embedding(self): + """Backward compat: no router -> bare litellm.aembedding as before""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + + mock_bare = AsyncMock(return_value=Mock(data=[{"embedding": [0.6, 0.7]}])) + with patch("litellm.aembedding", new=mock_bare): + _, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + ) + + mock_bare.assert_awaited_once_with(model="text-embedding-3-small", input=["test query"]) + assert request_body["queryVector"]["float32"] == [0.6, 0.7] + + def test_transform_search_uses_router_for_virtual_model_sync(self): + """Sync twin: router-served embedding model resolves via router.embedding""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["my-embedding-model"], sync=True) + + with patch("litellm.embedding", new=MagicMock()) as mock_bare_embedding: + _, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "my-embedding-model"}, + extra_body=None, + router=router, + ) + + router.embedding.assert_called_once_with(model="my-embedding-model", input=["test query"]) + mock_bare_embedding.assert_not_called() + assert request_body["queryVector"]["float32"] == [0.1, 0.2, 0.3] + + def test_transform_search_without_router_uses_bare_embedding_sync(self): + """Sync twin: no router -> bare litellm.embedding as before""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + + mock_bare = MagicMock(return_value=Mock(data=[{"embedding": [0.8, 0.9]}])) + with patch("litellm.embedding", new=mock_bare): + _, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + ) + + mock_bare.assert_called_once_with(model="text-embedding-3-small", input=["test query"]) + assert request_body["queryVector"]["float32"] == [0.8, 0.9] def test_transform_search_request_invalid_vector_store_id(self): """Test that invalid vector_store_id format raises error""" diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 15a117bd6fc..8bd67754952 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -327,3 +327,106 @@ def test_rag_query_stream_returns_event_stream(client_internal_user): assert response.headers.get("content-type", "").startswith("text/event-stream") assert '"object":"chat.completion.chunk"' in response.text assert "data: [DONE]" in response.text + + +def test_rag_query_merges_managed_store_params(client_internal_user): + """ + Regression: /v1/rag/query must consult the managed vector store registry + (like the direct /v1/vector_stores/{id}/search endpoint does) so that + provider, region, embedding model, etc. don't have to be repeated in + retrieval_config. Pre-fix the registry was never read, so managed S3 + Vectors stores failed with "aws_region_name is required". + """ + import litellm + from litellm.types.utils import ModelResponse + + mock_vector_store = { + "vector_store_id": "s3-store", + "custom_llm_provider": "s3_vectors", + "litellm_params": { + "aws_region_name": "eu-west-1", + "embedding_model": "my-embed", + "vector_bucket_name": "bkt", + }, + } + mock_registry = MagicMock() + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = mock_vector_store + + mock_response = ModelResponse( + id="chatcmpl-test", + choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + model="gpt-4o-mini", + ) + + with patch( + "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( + "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", + new=AsyncMock(), + ), patch( + "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", + new=AsyncMock(), + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store"}, + }, + ) + + assert response.status_code == 200, response.json() + mock_aquery.assert_awaited_once() + forwarded_config = mock_aquery.await_args.kwargs["retrieval_config"] + assert forwarded_config["vector_store_id"] == "s3-store" + assert forwarded_config["custom_llm_provider"] == "s3_vectors" + assert forwarded_config["aws_region_name"] == "eu-west-1" + assert forwarded_config["embedding_model"] == "my-embed" + assert forwarded_config["vector_bucket_name"] == "bkt" + + +def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): + """User-supplied retrieval_config keys must win over registry values.""" + import litellm + from litellm.types.utils import ModelResponse + + mock_vector_store = { + "vector_store_id": "s3-store", + "custom_llm_provider": "s3_vectors", + "litellm_params": {"aws_region_name": "eu-west-1"}, + } + mock_registry = MagicMock() + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = mock_vector_store + + mock_response = ModelResponse( + id="chatcmpl-test", + choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + model="gpt-4o-mini", + ) + + with patch( + "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( + "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", + new=AsyncMock(), + ), patch( + "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", + new=AsyncMock(), + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store", "aws_region_name": "us-east-1"}, + }, + ) + + assert response.status_code == 200, response.json() + forwarded_config = mock_aquery.await_args.kwargs["retrieval_config"] + assert forwarded_config["aws_region_name"] == "us-east-1" diff --git a/tests/test_litellm/rag/test_main.py b/tests/test_litellm/rag/test_main.py index 584124ba06a..d8ffae667b1 100644 --- a/tests/test_litellm/rag/test_main.py +++ b/tests/test_litellm/rag/test_main.py @@ -254,6 +254,96 @@ async def test_aquery_streaming_bills_sub_call_costs_into_final_event(): assert standard_logging_object["response_cost"] >= 0.003 +@pytest.mark.asyncio +async def test_aquery_forwards_provider_retrieval_config_and_router_to_search(): + """ + Regression: provider-specific retrieval_config keys (aws_region_name, + embedding_model, vector_bucket_name, ...) and the router must be forwarded + to the vector store search call. Pre-fix they were silently dropped, so + /v1/rag/query failed with provider config errors (e.g. S3 Vectors + "aws_region_name is required") even when the caller supplied them. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}, + } + ] + ) + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): + response = await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={ + "vector_store_id": "bkt:idx", + "custom_llm_provider": "s3_vectors", + "top_k": 5, + "aws_region_name": "eu-west-1", + "embedding_model": "my-embed", + "vector_bucket_name": "bkt", + }, + router=router, + mock_response="hi", + ) + + assert isinstance(response, ModelResponse) + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["vector_store_id"] == "bkt:idx" + assert search_kwargs["custom_llm_provider"] == "s3_vectors" + assert search_kwargs["max_num_results"] == 5 + assert search_kwargs["router"] is router + # provider-specific extras forwarded + assert search_kwargs["aws_region_name"] == "eu-west-1" + assert search_kwargs["embedding_model"] == "my-embed" + assert search_kwargs["vector_bucket_name"] == "bkt" + # consumed keys are not duplicated into the spread + assert "top_k" not in search_kwargs + + +@pytest.mark.asyncio +async def test_aquery_minimal_retrieval_config_forwards_no_extras(): + """ + A minimal retrieval_config must not leak consumed keys (or invent extras) + into the vector store search call. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): + await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={"vector_store_id": "vs_test_123", "custom_llm_provider": "openai"}, + mock_response="hi", + ) + + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["vector_store_id"] == "vs_test_123" + assert search_kwargs["custom_llm_provider"] == "openai" + assert search_kwargs["router"] is None + leaked = {"top_k", "filters", "retrieval_filter", "aws_region_name", "embedding_model", "vector_bucket_name"} + assert not (leaked & set(search_kwargs.keys())) + + def test_rag_call_types_are_registered(): """ query/aquery/ingest/aingest are @client-decorated entry points, so their diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index a9e5b3316e0..91d2973af76 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5936,3 +5936,58 @@ async def test_acreate_batch_request_bedrock_tags_override_deployment_tags(): bedrock_tags=request_tags, ) assert mock_sign.call_args.kwargs["data"]["tags"] == request_tags + + +@pytest.mark.asyncio +async def test_avector_store_search_injects_router(): + """ + Regression: router.avector_store_search must pass the router down to the + SDK search call so provider transforms can resolve router-managed + embedding models (e.g. S3 Vectors query embeddings). + """ + from litellm.types.vector_stores import VectorStoreSearchResponse + + mock_asearch = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + # Router.__init__ binds asearch via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.asearch", new=mock_asearch): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + await router.avector_store_search( + vector_store_id="v", query="q", custom_llm_provider="s3_vectors" + ) + + mock_asearch.assert_awaited_once() + assert mock_asearch.await_args.kwargs["router"] is router + + +@pytest.mark.asyncio +async def test_avector_store_create_does_not_inject_router(): + """The router injection is gated on the search call type: the create path + must keep calling the SDK without a router kwarg.""" + mock_acreate = AsyncMock(return_value={"id": "vs_1", "object": "vector_store"}) + # avector_store_create(model=None) resolves acreate via a local import at + # call time, so patching after Router construction works here. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + with patch("litellm.vector_stores.main.acreate", new=mock_acreate): + await router.avector_store_create(model=None, custom_llm_provider="openai") + + mock_acreate.assert_awaited_once() + assert "router" not in mock_acreate.await_args.kwargs diff --git a/tests/test_litellm/vector_stores/test_main.py b/tests/test_litellm/vector_stores/test_main.py new file mode 100644 index 00000000000..3fdf4d9daa5 --- /dev/null +++ b/tests/test_litellm/vector_stores/test_main.py @@ -0,0 +1,77 @@ +""" +Tests for litellm/vector_stores/main.py. + +Pins the router threading contract for vector store search: the router is an +explicit named parameter that reaches the HTTP handler, and it must never leak +into litellm_params/kwargs where logging would model_dump() it (the #19550 +serialization trap). +""" + +from unittest.mock import MagicMock, patch + +import litellm.vector_stores.main as vector_stores_main +from litellm.vector_stores.main import search + +MOCK_SEARCH_RESPONSE = { + "object": "vector_store.search_results.page", + "search_query": "q", + "data": [], +} + + +def test_search_threads_router_to_handler(): + """search() must pass its router param through to the HTTP handler""" + mock_router = MagicMock() + logger = MagicMock() + + with ( + patch( + "litellm.vector_stores.main.ProviderConfigManager.get_provider_vector_stores_config", + return_value=MagicMock(), + ), + patch.object( + vector_stores_main.base_llm_http_handler, + "vector_store_search_handler", + return_value=MOCK_SEARCH_RESPONSE, + ) as mock_handler, + ): + search( + vector_store_id="bkt:idx", + query="q", + custom_llm_provider="s3_vectors", + router=mock_router, + litellm_logging_obj=logger, + ) + + mock_handler.assert_called_once() + assert mock_handler.call_args.kwargs["router"] is mock_router + + +def test_search_router_not_in_litellm_params(): + """Regression (#19550 class): the router must stay out of GenericLiteLLMParams, + otherwise pre-call logging model_dump()s it and breaks serialization.""" + mock_router = MagicMock() + logger = MagicMock() + + with ( + patch( + "litellm.vector_stores.main.ProviderConfigManager.get_provider_vector_stores_config", + return_value=MagicMock(), + ), + patch.object( + vector_stores_main.base_llm_http_handler, + "vector_store_search_handler", + return_value=MOCK_SEARCH_RESPONSE, + ) as mock_handler, + ): + search( + vector_store_id="bkt:idx", + query="q", + custom_llm_provider="s3_vectors", + router=mock_router, + litellm_logging_obj=logger, + ) + + litellm_params = mock_handler.call_args.kwargs["litellm_params"] + assert "router" not in litellm_params.model_dump(exclude_none=True) + assert getattr(litellm_params, "router", None) is None diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx index cbabcc6dca5..f375bfdd351 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx @@ -128,16 +128,33 @@ describe("VectorStoreTester", () => { await waitFor(() => expect(mockSearch).toHaveBeenCalledTimes(1)); }); - it("reports a failed search and keeps the history empty", async () => { + it("shows the backend error in the history when a search fails", async () => { const user = userEvent.setup(); - mockSearch.mockRejectedValue(new Error("boom")); + const errorBody = '{"error":{"message":"OpenAIException - api_key is required"}}'; + mockSearch.mockRejectedValue(new Error(errorBody)); renderTester(); await user.type(queryInput(), "hello"); await user.click(searchButton()); - await waitFor(() => expect(mockFromBackend).toHaveBeenCalledWith("Failed to search vector store")); - expect(screen.getByText(EMPTY_STATE)).toBeInTheDocument(); + await waitFor(() => expect(mockFromBackend).toHaveBeenCalledWith(errorBody)); + expect(screen.getByText(`Search failed: ${errorBody}`)).toBeInTheDocument(); + expect(screen.queryByText("No results found")).not.toBeInTheDocument(); + expect(screen.queryByText(EMPTY_STATE)).not.toBeInTheDocument(); + // the failed query stays in the input for retry + expect(queryInput()).toHaveValue("hello"); + }); + + it('renders "No results found" for an empty result set, not an error', async () => { + const user = userEvent.setup(); + mockSearch.mockResolvedValue({ object: "vector_store.search_results.page", search_query: "hello", data: [] }); + renderTester(); + + await user.type(queryInput(), "hello"); + await user.click(searchButton()); + + expect(await screen.findByText("No results found")).toBeInTheDocument(); + expect(screen.queryByText(/search failed/i)).not.toBeInTheDocument(); }); it("clears the search history", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx index 015d58e8649..65b31bb911a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx @@ -41,6 +41,7 @@ export const VectorStoreTester: React.FC = ({ vectorStor { query: string; response: VectorStoreSearchResponse | null; + error: string | null; timestamp: number; }[] >([]); @@ -60,6 +61,7 @@ export const VectorStoreTester: React.FC = ({ vectorStor const historyEntry = { query, response, + error: null, timestamp: Date.now(), }; @@ -67,7 +69,9 @@ export const VectorStoreTester: React.FC = ({ vectorStor setQuery(""); } catch (error) { console.error("Error searching vector store:", error); - NotificationsManager.fromBackend("Failed to search vector store"); + const errorMessage = error instanceof Error ? error.message : String(error); + NotificationsManager.fromBackend(errorMessage); + setSearchHistory((prev) => [{ query, response: null, error: errorMessage, timestamp: Date.now() }, ...prev]); } finally { setIsLoading(false); } @@ -228,6 +232,8 @@ export const VectorStoreTester: React.FC = ({ vectorStor ); })} + ) : entry.error ? ( +
Search failed: {entry.error}
) : (
No results found
)} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 576e16cbb37..a8408fecaad 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -6851,7 +6851,7 @@ export const vectorStoreSearchCall = async ( if (!response.ok) { const errorData = await response.text(); await handleError(errorData); - return null; + throw new Error(errorData); } const data = await response.json(); From 9e25dd708fa7a8b2af354e6901683fd604c4a19a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:47:13 -0700 Subject: [PATCH 02/67] feat(streaming): carry final response cost on streamed usage by default Streamed responses through the proxy previously exposed no usable cost: the x-litellm-response-cost header is unreadable mid-stream and the final usage chunk carried only tokens, priced against an alias model name the client cannot resolve. The include_cost_in_streaming_usage flag existed but was off by default and only fixed the wire, not SDK clients. Stamp usage.cost into the joined streaming response by default wherever a final usage object is built: the chat-completions stream_chunk_builder, the native /v1/responses RESPONSE_COMPLETED event, and synthetic response events. Provider-reported cost always wins over the computed value, and only positive computed costs are stamped so unpriceable alias responses keep deferring to the logging object's own calculation. Per-chunk SSE cost injection (/v1/messages, generateContent, passthrough) stays behind the flag. Also normalize non-litellm usage objects in stream_chunk_builder: openai CompletionUsage lacks Usage.__contains__, so membership probes silently returned False and client-side rebuilds dropped the wire cost and recounted token usage locally. Wire token counts and cost now survive. Resolves LIT-6427 --- .../streaming_chunk_builder_utils.py | 8 ++- litellm/main.py | 22 ++++--- .../streaming_iterator.py | 10 --- litellm/responses/streaming_iterator.py | 46 ++++++------- .../test_streaming_chunk_builder_utils.py | 53 +++++++++++++++ .../responses/test_streaming_iterator.py | 52 +++++++++++++++ tests/test_litellm/test_main.py | 66 +++++++++++++++++-- 7 files changed, 204 insertions(+), 53 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 0e2139d688b..276616f9eee 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -36,6 +36,8 @@ from litellm.types.utils import ( from litellm.utils import print_verbose, token_counter if TYPE_CHECKING: + from openai.types.completion_usage import CompletionUsage + from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.litellm_core_utils.streaming_chunk_builder_utils import ( UsagePerChunk, @@ -782,7 +784,7 @@ class ChunkProcessor: @staticmethod def _extract_usage_chunk(chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream") -> Usage | None: - usage_chunk: Usage | None = None + usage_chunk: Usage | CompletionUsage | None = None if hasattr(chunk, "usage") and chunk.usage is not None: usage_chunk = chunk.usage elif "usage" in chunk: @@ -794,7 +796,9 @@ class ChunkProcessor: if isinstance(usage_chunk, dict): return Usage(**usage_chunk) - return usage_chunk + if usage_chunk is None or isinstance(usage_chunk, Usage): + return usage_chunk + return Usage(**usage_chunk.model_dump()) def _calculate_usage_per_chunk( self, diff --git a/litellm/main.py b/litellm/main.py index 0c8bff16f81..7ca84226b09 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8634,6 +8634,16 @@ def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Opti hidden_params["response_cost"] = response_cost +def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: + if logging_obj is None: + return + if isinstance(getattr(usage, "cost", None), (int, float)): + return + computed_cost: Final = logging_obj._response_cost_calculator(result=response) + if isinstance(computed_cost, (int, float)) and computed_cost > 0: + setattr(usage, "cost", computed_cost) + + def stream_chunk_builder( chunks: list, messages: list | None = None, @@ -8728,12 +8738,7 @@ def stream_chunk_builder( ) break - if litellm.include_cost_in_streaming_usage and logging_obj is not None: - setattr( - usage, - "cost", - logging_obj._response_cost_calculator(result=response), - ) + _stamp_streaming_usage_cost(usage, response, logging_obj) _set_stream_builder_response_cost(response, logging_obj) processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) @@ -8912,10 +8917,7 @@ def stream_chunk_builder( ) break - # Add cost to usage object if include_cost_in_streaming_usage is True - if litellm.include_cost_in_streaming_usage and logging_obj is not None: - setattr(usage, "cost", logging_obj._response_cost_calculator(result=response)) - + _stamp_streaming_usage_cost(usage, response, logging_obj) _set_stream_builder_response_cost(response, logging_obj) processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 8b1eeb30306..27afff39c0f 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1164,16 +1164,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _emit_response_completed_event(self, litellm_model_response: ModelResponse) -> ResponseCompletedEvent | None: if litellm_model_response: - # Add cost to usage object if include_cost_in_streaming_usage is True - if litellm.include_cost_in_streaming_usage and self.litellm_logging_obj is not None: - usage: Final[object] = getattr(litellm_model_response, "usage", None) - if usage is not None: - setattr( - usage, - "cost", - self.litellm_logging_obj._response_cost_calculator(result=litellm_model_response), - ) - # Transform the response responses_api_response: Final = ( LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index d070f7758fd..2b4252aa1c5 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -405,23 +405,7 @@ class BaseResponsesAPIStreamingIterator: openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED, ): self.completed_response = openai_responses_api_chunk - # Add cost to usage object if include_cost_in_streaming_usage is True - if litellm.include_cost_in_streaming_usage and self.logging_obj is not None: - response_obj: Final[ResponsesAPIResponse | None] = getattr( - openai_responses_api_chunk, "response", None - ) - if response_obj: - usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None) - if usage_obj is not None: - try: - cost: Final[float | None] = self.logging_obj._response_cost_calculator( - result=response_obj - ) - if cost is not None: - setattr(usage_obj, "cost", cost) - except Exception: - # Best-effort usage cost annotation should not break stream replay. - pass + _stamp_responses_usage_cost(getattr(openai_responses_api_chunk, "response", None), self.logging_obj) if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED: self._handle_logging_failed_response() @@ -1272,6 +1256,24 @@ def _add_text_like_part_events( ) +def _stamp_responses_usage_cost( + response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None +) -> None: + if response_obj is None or logging_obj is None: + return + usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None) + if usage_obj is None: + return + if isinstance(getattr(usage_obj, "cost", None), (int, float)): + return + try: + cost: Final[float | None] = logging_obj._response_cost_calculator(result=response_obj) + except Exception: + return + if isinstance(cost, (int, float)) and cost > 0: + setattr(usage_obj, "cost", cost) + + def _build_synthetic_response_events( *, transformed: ResponsesAPIResponse, @@ -1279,15 +1281,7 @@ def _build_synthetic_response_events( chunk_size: int, ) -> list[ResponsesAPIStreamingResponse]: openai_types: Final = _get_openai_response_types() - if litellm.include_cost_in_streaming_usage and logging_obj is not None: - usage_obj: Final = transformed.usage if hasattr(transformed, "usage") else None - if usage_obj is not None: - try: - cost: Final[float | None] = logging_obj._response_cost_calculator(result=transformed) - if cost is not None: - setattr(usage_obj, "cost", cost) - except Exception: - pass + _stamp_responses_usage_cost(transformed, logging_obj) events: Final[list[ResponsesAPIStreamingResponse]] = [ _build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed), diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 8ac050a04f9..bacbcbf132b 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -592,6 +592,59 @@ def test_stream_chunk_builder_litellm_usage_chunks(): assert usage.total_tokens == 77 +def test_calculate_usage_honors_openai_sdk_completion_usage_chunks(): + from openai.types.completion_usage import CompletionUsage + + content_chunk = ModelResponseStream( + id="chatcmpl-sdk-usage-1", + created=1745513206, + model="mantle-claude", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta( + provider_specific_fields=None, + content="ok", + role=None, + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields=None, + stream_options={"include_usage": True}, + ) + usage_chunk = ModelResponseStream( + id="chatcmpl-sdk-usage-1", + created=1745513207, + model="mantle-claude", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[], + provider_specific_fields=None, + stream_options={"include_usage": True}, + ) + usage_chunk.usage = CompletionUsage( + prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704 + ) + assert type(usage_chunk.usage) is CompletionUsage + + chunks = [content_chunk, usage_chunk] + usage = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, model="mantle-claude", completion_output="" + ) + + assert usage.prompt_tokens == 20 + assert usage.completion_tokens == 60 + assert usage.total_tokens == 80 + assert getattr(usage, "cost", None) == pytest.approx(0.000704) + + def test_get_model_from_chunks_azure_model_router(): """ Test that _get_model_from_chunks finds the actual model from Azure Model Router chunks. diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index 677faf7f655..9edcaaef034 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -326,3 +326,55 @@ def test_run_post_success_hooks_does_not_report_generation_time_as_overhead(): assert iterator.completed_response._hidden_params["_response_ms"] == 10000.0 assert "litellm_overhead_time_ms" not in iterator.completed_response._hidden_params + + +def _responses_api_response_with_usage() -> ResponsesAPIResponse: + from litellm.types.llms.openai import ResponseAPIUsage + + return ResponsesAPIResponse( + id="resp_lit6427", + created_at=int(datetime(2025, 1, 1).timestamp()), + status="completed", + model="mantle-claude", + object="response", + output=[], + usage=ResponseAPIUsage(input_tokens=20, output_tokens=60, total_tokens=80), + ) + + +def test_stamp_responses_usage_cost_stamps_computed_cost(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + + response = _responses_api_response_with_usage() + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj._response_cost_calculator.return_value = 0.000704 + + _stamp_responses_usage_cost(response, logging_obj) + + assert getattr(response.usage, "cost", None) == pytest.approx(0.000704) + logging_obj._response_cost_calculator.assert_called_once_with(result=response) + + +def test_stamp_responses_usage_cost_keeps_provider_reported_cost(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + + response = _responses_api_response_with_usage() + setattr(response.usage, "cost", 0.5) + logging_obj = Mock(spec=LiteLLMLoggingObj) + + _stamp_responses_usage_cost(response, logging_obj) + + assert getattr(response.usage, "cost", None) == pytest.approx(0.5) + logging_obj._response_cost_calculator.assert_not_called() + + +def test_stamp_responses_usage_cost_survives_calculator_failure(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + + response = _responses_api_response_with_usage() + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj._response_cost_calculator.side_effect = RuntimeError("cost map unavailable") + + _stamp_responses_usage_cost(response, logging_obj) + + assert getattr(response.usage, "cost", None) is None diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 8cf878d05d9..7c2b9d0be05 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -3150,8 +3150,8 @@ def _stream_builder_logging_obj() -> LiteLLMLogging: return logging_obj -def test_stream_chunk_builder_reports_streaming_usage_cost_when_enabled(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) +def test_stream_chunk_builder_stamps_streaming_usage_cost_by_default(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) chunks: Final = [ _stream_builder_text_chunk("gpt-4o", "Hello "), _stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"), @@ -3168,11 +3168,45 @@ def test_stream_chunk_builder_reports_streaming_usage_cost_when_enabled(monkeypa assert response._hidden_params["response_cost"] == pytest.approx(usage_cost) -def test_stream_chunk_builder_defers_cost_to_logging_obj_when_usage_cost_absent(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) +def test_stream_chunk_builder_skips_stamp_when_cost_is_unpriceable(): + import time as time_module + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + logging_obj: Final = LiteLLMLogging( + model="us.anthropic.claude-opus-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="completion", + start_time=time_module.time(), + litellm_call_id="stream-builder-alias-unpriceable", + function_id="1", + ) + logging_obj.model_call_details["custom_llm_provider"] = "bedrock" + logging_obj.optional_params = {} + usage_chunk: Final = _stream_builder_text_chunk("bedrock-claude-opus-5", "") + usage_chunk.usage = Usage(prompt_tokens=40, completion_tokens=5, total_tokens=45) + chunks: Final = [ + _stream_builder_text_chunk("bedrock-claude-opus-5", "Hello ", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=logging_obj + ) + + assert response is not None + assert getattr(response.usage, "cost", None) is None + assert response._hidden_params.get("response_cost") is None + + +def test_stream_chunk_builder_keeps_provider_reported_usage_cost(): + usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "") + usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5) chunks: Final = [ _stream_builder_text_chunk("gpt-4o", "Hello "), _stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"), + usage_chunk, ] response: Final = litellm.stream_chunk_builder( @@ -3180,4 +3214,26 @@ def test_stream_chunk_builder_defers_cost_to_logging_obj_when_usage_cost_absent( ) assert response is not None - assert response._hidden_params.get("response_cost") is None + assert getattr(response.usage, "cost", None) == pytest.approx(0.5) + assert response._hidden_params["response_cost"] == pytest.approx(0.5) + + +def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk(): + from openai.types.completion_usage import CompletionUsage + + usage_chunk: Final = _stream_builder_text_chunk("mantle-claude", "") + usage_chunk.usage = CompletionUsage(prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704) + assert type(usage_chunk.usage) is CompletionUsage + chunks: Final = [ + _stream_builder_text_chunk("mantle-claude", "Hello "), + _stream_builder_text_chunk("mantle-claude", "world.", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder(chunks=chunks, messages=[{"role": "user", "content": "hi"}]) + + assert response is not None + assert response.usage.prompt_tokens == 20 + assert response.usage.completion_tokens == 60 + assert getattr(response.usage, "cost", None) == pytest.approx(0.000704) + assert response._hidden_params["response_cost"] == pytest.approx(0.000704) From 7ca035f310f891eea216f3162191739f3720bcb1 Mon Sep 17 00:00:00 2001 From: Kris Xia Date: Tue, 1 Sep 2026 11:39:33 +0800 Subject: [PATCH 03/67] fix(gemini): return enabled thinking content by default --- .../gemini/vertex_and_google_ai_studio_gemini.py | 2 +- .../test_vertex_and_google_ai_studio_gemini.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d8b1e7ba17c..69fe5678de9 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -949,7 +949,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # For Gemini 3+ models, use thinkingLevel instead of thinkingBudget if model and VertexGeminiConfig._is_gemini_3_or_newer(model): if thinking_enabled: - if thinking_budget is None or thinking_budget == 0: + if thinking_budget == 0: params["includeThoughts"] = False else: params["includeThoughts"] = True diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index bd07bec900f..d2788408e09 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1185,6 +1185,18 @@ def test_vertex_ai_map_thinking_param_with_budget_tokens_0(): } +def test_vertex_ai_map_thinking_param_without_budget_tokens_for_gemini_3(): + v = VertexGeminiConfig() + result = v.map_openai_params( + non_default_params={"thinking": {"type": "enabled"}}, + optional_params={}, + model="gemini-3.5-flash", + drop_params=False, + ) + + assert result["thinkingConfig"] == {"includeThoughts": True} + + def test_vertex_ai_map_tools(): v = VertexGeminiConfig() optional_params = {} From 24ee419c85f6758369e6582250a50490ce1d6819 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:28:04 +0000 Subject: [PATCH 04/67] fix(models): registry audit 2026-09-01 for openai realtime, mistral aliases, voyage, xai, fireworks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 133 ++++++++++++++---- model_prices_and_context_window.json | 133 ++++++++++++++---- 2 files changed, 212 insertions(+), 54 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 27ff525c15e..08add22c998 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -30606,17 +30606,18 @@ }, "gpt-realtime-2": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, - "output_cost_per_token": 1.6e-05, + "output_cost_per_token": 2.4e-05, "supported_endpoints": [ "/v1/realtime" ], @@ -30680,8 +30681,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -30713,7 +30714,7 @@ "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", - "max_input_tokens": 128000, + "max_input_tokens": 32000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", @@ -33477,19 +33478,21 @@ "source": "https://mistral.ai/pricing#api-pricing" }, "mistral/magistral-medium-latest": { - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-2506": { "deprecation_date": "2025-11-30", @@ -33508,19 +33511,21 @@ "supports_tool_choice": true }, "mistral/magistral-small-latest": { - "input_cost_per_token": 5e-07, + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", + "output_cost_per_token": 6e-07, + "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-1-2-2509": { "deprecation_date": "2026-07-31", @@ -33652,16 +33657,21 @@ "supports_vision": true }, "mistral/mistral-medium": { - "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 8.1e-06, + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/mistral-medium-2312": { "deprecation_date": "2025-06-16", @@ -45539,6 +45549,26 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, + "voyage/rerank-3": { + "input_cost_per_token": 5e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, + "voyage/rerank-3-lite": { + "input_cost_per_token": 2e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, "voyage/voyage-2": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", @@ -46790,6 +46820,27 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-build-latest": { + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "xai", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://docs.x.ai/developers/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "xai/grok-4.6": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, @@ -57110,6 +57161,34 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/glm-5p3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/accounts/fireworks/models/inkling": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 4.05e-06, + "source": "https://fireworks.ai/models/fireworks/inkling", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-8b": { "input_cost_per_token": 1e-07, "output_cost_per_token": 0.0, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 27ff525c15e..08add22c998 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -30606,17 +30606,18 @@ }, "gpt-realtime-2": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, - "output_cost_per_token": 1.6e-05, + "output_cost_per_token": 2.4e-05, "supported_endpoints": [ "/v1/realtime" ], @@ -30680,8 +30681,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -30713,7 +30714,7 @@ "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", - "max_input_tokens": 128000, + "max_input_tokens": 32000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", @@ -33477,19 +33478,21 @@ "source": "https://mistral.ai/pricing#api-pricing" }, "mistral/magistral-medium-latest": { - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-2506": { "deprecation_date": "2025-11-30", @@ -33508,19 +33511,21 @@ "supports_tool_choice": true }, "mistral/magistral-small-latest": { - "input_cost_per_token": 5e-07, + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", + "output_cost_per_token": 6e-07, + "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-1-2-2509": { "deprecation_date": "2026-07-31", @@ -33652,16 +33657,21 @@ "supports_vision": true }, "mistral/mistral-medium": { - "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 8.1e-06, + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/mistral-medium-2312": { "deprecation_date": "2025-06-16", @@ -45539,6 +45549,26 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, + "voyage/rerank-3": { + "input_cost_per_token": 5e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, + "voyage/rerank-3-lite": { + "input_cost_per_token": 2e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, "voyage/voyage-2": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", @@ -46790,6 +46820,27 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-build-latest": { + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "xai", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://docs.x.ai/developers/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "xai/grok-4.6": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, @@ -57110,6 +57161,34 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/glm-5p3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/accounts/fireworks/models/inkling": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 4.05e-06, + "source": "https://fireworks.ai/models/fireworks/inkling", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-8b": { "input_cost_per_token": 1e-07, "output_cost_per_token": 0.0, From cf4738c3b736edb62f111ab26d899da6a7f9f284 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:32:04 -0700 Subject: [PATCH 05/67] style: format s3 vectors transformation and rag endpoints --- litellm/llms/s3_vectors/vector_stores/transformation.py | 4 +++- litellm/proxy/rag_endpoints/endpoints.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/llms/s3_vectors/vector_stores/transformation.py b/litellm/llms/s3_vectors/vector_stores/transformation.py index 3c1b0025d08..733358381fe 100644 --- a/litellm/llms/s3_vectors/vector_stores/transformation.py +++ b/litellm/llms/s3_vectors/vector_stores/transformation.py @@ -68,7 +68,9 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): """Return the router iff it serves ``embedding_model`` as a deployment.""" if router is None: return None - model_list: Final = [dict(m) for m in (router.get_model_list() or ())] # mutable-ok: resolve_embedding_router requires list[dict] + model_list: Final = [ + dict(m) for m in (router.get_model_list() or ()) + ] # mutable-ok: resolve_embedding_router requires list[dict] return resolve_embedding_router(embedding_model=embedding_model, llm_router=router, llm_model_list=model_list) def transform_search_vector_store_request( diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index 31998d2040e..5c392c30018 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -717,7 +717,10 @@ async def rag_query( vector_store_id=retrieval_config["vector_store_id"], user_api_key_dict=user_api_key_dict, ) - merged_retrieval_config: Final = {**store_data, **retrieval_config} # mutable-ok: litellm.aquery requires a plain dict payload + merged_retrieval_config: Final = { + **store_data, + **retrieval_config, + } # mutable-ok: litellm.aquery requires a plain dict payload # Add litellm data request_data: dict[str, object] = {} From d3dab8e294b06badb2d34f31ce6aade9380a49ea Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:46:04 -0700 Subject: [PATCH 06/67] fix(rerank): map provider errors with the resolved provider on sync and async paths --- litellm/rerank_api/main.py | 22 ++++++-- tests/test_litellm/rerank_api/test_main.py | 61 ++++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index c8f7842aebf..597d1cfb863 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -43,10 +43,17 @@ async def arerank( """ Async: Reranks a list of documents based on their relevance to the query """ + _custom_llm_provider: str | None = None # rebind-ok: set by the get_llm_provider unpack; read in the except try: loop: Final = asyncio.get_event_loop() kwargs["arerank"] = True + _, _custom_llm_provider, _, _ = litellm.get_llm_provider( # rebind-ok: see pre-declaration above + model=model, + custom_llm_provider=custom_llm_provider, + api_base=kwargs.get("api_base", None), + ) + func: Final = partial( rerank, model, @@ -70,7 +77,11 @@ async def arerank( response = init_response return response except Exception as e: - raise e + raise exception_type( + model=model, + custom_llm_provider=_custom_llm_provider or custom_llm_provider, + original_exception=e, + ) @client @@ -115,6 +126,7 @@ def rerank( model_info: Final = kwargs.get("model_info", None) user: Final = kwargs.get("user", None) client: Final = kwargs.get("client", None) + _custom_llm_provider: str | None = None # rebind-ok: set by the get_llm_provider unpack; read in the except try: _is_async: Final = kwargs.pop("arerank", False) is True optional_params: Final = GenericLiteLLMParams(**kwargs) @@ -127,7 +139,7 @@ def rerank( ( model, - _custom_llm_provider, + _custom_llm_provider, # rebind-ok: see pre-declaration above dynamic_api_key, dynamic_api_base, ) = litellm.get_llm_provider( @@ -538,4 +550,8 @@ def rerank( return response except Exception as e: verbose_logger.error("Error in rerank: %s", e) - raise exception_type(model=model, custom_llm_provider=custom_llm_provider, original_exception=e) + raise exception_type( + model=model, + custom_llm_provider=_custom_llm_provider or custom_llm_provider, + original_exception=e, + ) diff --git a/tests/test_litellm/rerank_api/test_main.py b/tests/test_litellm/rerank_api/test_main.py index 587be59c550..62149c742d6 100644 --- a/tests/test_litellm/rerank_api/test_main.py +++ b/tests/test_litellm/rerank_api/test_main.py @@ -111,6 +111,67 @@ def test_together_rerank_honors_api_base(respx_mock: respx.MockRouter): assert mock_route.calls[0].request.headers["authorization"] == "Bearer fake-together-key" +DASHSCOPE_404_BODY = { + "error": { + "message": "The model `does-not-exist` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": None, + "code": "model_not_found", + }, + "request_id": "mock-request-id", +} + + +def test_rerank_error_names_provider_and_keeps_body(respx_mock: respx.MockRouter, monkeypatch): + """Regression for the rerank error path mapping with the unresolved provider param: + a provider 404 surfaced as 'None - ' instead of naming the provider and its error body.""" + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False) + + mock_route = respx_mock.post("https://dashscope.example/v1/reranks") + mock_route.return_value = httpx.Response(404, json=DASHSCOPE_404_BODY) + + with pytest.raises(litellm.NotFoundError) as exc_info: + litellm.rerank( + model="dashscope/does-not-exist", + query=MARKER_QUERY, + documents=[MARKER_DOC], + api_key="fake-dashscope-key", + api_base="https://dashscope.example/v1", + ) + + assert mock_route.called + assert "DashscopeException" in str(exc_info.value) + assert "does not exist or you do not have access to it" in str(exc_info.value) + assert "None - " not in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_arerank_error_is_mapped_to_litellm_exception(respx_mock: respx.MockRouter, monkeypatch): + """Regression for arerank's bare re-raise: provider errors escaped as raw + provider exception classes instead of the mapped litellm exception contract.""" + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + + mock_route = respx_mock.post("https://dashscope.example/v1/reranks") + mock_route.return_value = httpx.Response(404, json=DASHSCOPE_404_BODY) + + with pytest.raises(litellm.NotFoundError) as exc_info: + await litellm.arerank( + model="dashscope/does-not-exist", + query=MARKER_QUERY, + documents=[MARKER_DOC], + api_key="fake-dashscope-key", + api_base="https://dashscope.example/v1", + ) + + assert mock_route.called + assert "DashscopeException" in str(exc_info.value) + assert "does not exist or you do not have access to it" in str(exc_info.value) + assert "None - " not in str(exc_info.value) + + @pytest.mark.asyncio async def test_together_rerank_async_honors_env_api_base(respx_mock: respx.MockRouter, monkeypatch): """Regression: TOGETHER_AI_API_BASE was honored by chat but ignored by rerank.""" From 3914de24eff6c0f3deda46ec2f2217cb0305d916 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:52:45 -0700 Subject: [PATCH 07/67] fix(router): route model-less sync vector store calls to the SDK _generic_api_call_with_fallbacks requires a model, so sync vector_store_search and vector_store_create raised a TypeError whenever the call carried no model. Model-less calls now go directly to the SDK function, with the router injected for search, matching the async wrapper's behavior --- litellm/router.py | 19 +++++++---- tests/test_litellm/test_router.py | 55 +++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d8769525815..96d5b1e1488 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6299,8 +6299,6 @@ class Router: "responses", "generate_content", "generate_content_stream", - "vector_store_search", - "vector_store_create", "ocr", "search", "video_generation", @@ -6324,6 +6322,8 @@ class Router: return sync_wrapper if call_type in ( + "vector_store_search", + "vector_store_create", "vector_store_retrieve", "vector_store_list", "vector_store_update", @@ -6335,11 +6335,16 @@ class Router: client: object | None = None, **kwargs, ): - if custom_llm_provider and "custom_llm_provider" not in kwargs: - kwargs["custom_llm_provider"] = custom_llm_provider - if kwargs.get("model"): - return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs) - return original_function(**kwargs) + provider_kwargs: Final = ( + MappingProxyType({**kwargs, "custom_llm_provider": custom_llm_provider}) + if custom_llm_provider and "custom_llm_provider" not in kwargs + else MappingProxyType(kwargs) + ) + if provider_kwargs.get("model"): + return self._generic_api_call_with_fallbacks(original_function=original_function, **provider_kwargs) + if call_type == "vector_store_search": + return original_function(**MappingProxyType({**provider_kwargs, "router": self})) + return original_function(**provider_kwargs) return vector_store_sync_wrapper diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c222fab79f0..3abfe8ce522 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7567,6 +7567,61 @@ async def test_avector_store_create_does_not_inject_router(): assert "router" not in mock_acreate.await_args.kwargs +def test_vector_store_search_injects_router(): + """ + Sync parity for the router injection: router.vector_store_search must pass + the router down to the SDK search call so provider transforms can resolve + router-managed embedding models, same as avector_store_search. + """ + from litellm.types.vector_stores import VectorStoreSearchResponse + + expected_response = VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + mock_search = MagicMock(return_value=expected_response) + # Router.__init__ binds search via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.search", new=mock_search): # test-quality-ok: the SDK call is the only place the injected router kwarg is observable + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + search_response = router.vector_store_search( + vector_store_id="v", query="q", custom_llm_provider="s3_vectors" + ) + + assert search_response is expected_response + mock_search.assert_called_once() + assert mock_search.call_args.kwargs["router"] is router + assert mock_search.call_args.kwargs["custom_llm_provider"] == "s3_vectors" + + +def test_vector_store_create_does_not_inject_router(): + """The sync create path must keep calling the SDK without a router kwarg.""" + expected_response = {"id": "vs_1", "object": "vector_store"} + mock_create = MagicMock(return_value=expected_response) + # Router.__init__ binds create via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.create", new=mock_create): # test-quality-ok: the SDK call is the only place a leaked router kwarg would surface + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + create_response = router.vector_store_create(custom_llm_provider="openai") + + assert create_response is expected_response + mock_create.assert_called_once() + assert "router" not in mock_create.call_args.kwargs + + class TestPreRoutingStrategyRegistryLifecycle: """ Regression tests: a deployment leaving the model_list must release the From babe7816ada8d622be993b542fc2512037d2466f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:52:45 -0700 Subject: [PATCH 08/67] fix(rag): store-wins merge, single lookup, allowlisted search params rag_query reuses the store resolved during authorization instead of a second registry lookup, merges registry data store-wins so callers cannot override a managed store's provider or credentials, and logs ids instead of the merged config, which can carry resolved credentials. aquery forwards only allowlisted retrieval_config keys to vector store search, keeping caller-supplied connection overrides like api_base and api_key away from the search call --- litellm/proxy/rag_endpoints/endpoints.py | 53 ++++++++---- .../proxy/vector_store_endpoints/endpoints.py | 86 ++++++++++--------- .../management_endpoints.py | 4 +- litellm/rag/main.py | 25 ++++-- .../proxy/rag_endpoints/test_rag_endpoints.py | 26 +++--- tests/test_litellm/rag/test_main.py | 39 +++++++++ 6 files changed, 149 insertions(+), 84 deletions(-) diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index 5c392c30018..d2c7d6f93ee 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -9,6 +9,7 @@ Provides: import base64 import json from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import orjson @@ -19,6 +20,9 @@ from starlette.datastructures import UploadFile import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + LiteLLM_ManagedVectorStore, +) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.proxy._types import * from litellm.proxy.auth.auth_utils import is_request_body_safe @@ -37,7 +41,7 @@ from litellm.proxy.rag_endpoints.upload_security import ( validate_upload, ) from litellm.proxy.vector_store_endpoints.endpoints import ( - _update_request_data_with_litellm_managed_vector_store_registry, # pyright: ignore[reportPrivateUsage] # shared registry-merge helper used by the direct search endpoint + build_request_data_from_managed_vector_store, ) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, @@ -123,12 +127,21 @@ def _collect_vector_store_ids_from_payload(payload: object) -> set[str]: async def _authorize_nested_vector_store_ids( payload: object, user_api_key_dict: UserAPIKeyAuth, -) -> None: - for vector_store_id in sorted(_collect_vector_store_ids_from_payload(payload)): - await assert_user_can_access_vector_store_id( - vector_store_id=vector_store_id, - user_api_key_dict=user_api_key_dict, - ) +) -> Mapping[str, LiteLLM_ManagedVectorStore]: + """Authorize every nested vector store id and return the managed stores it resolved.""" + return MappingProxyType( + { + vector_store_id: store + for vector_store_id in sorted(_collect_vector_store_ids_from_payload(payload)) + if ( + store := await assert_user_can_access_vector_store_id( + vector_store_id=vector_store_id, + user_api_key_dict=user_api_key_dict, + ) + ) + is not None + } + ) def _build_file_metadata_entry( @@ -703,23 +716,24 @@ async def rag_query( status_code=400, detail={"error": "retrieval_config must contain 'vector_store_id'"}, ) - await _authorize_nested_vector_store_ids( + resolved_stores: Final = await _authorize_nested_vector_store_ids( payload=retrieval_config, user_api_key_dict=user_api_key_dict, ) # Merge litellm-managed vector store params (provider, region, embedding - # model, credentials, ...) from the registry — same source the direct - # /vector_stores/{id}/search endpoint uses. User-supplied - # retrieval_config keys win on conflict. - store_data: Final = await _update_request_data_with_litellm_managed_vector_store_registry( - data={}, # mutable-ok: the helper mutates and returns the seed dict - vector_store_id=retrieval_config["vector_store_id"], - user_api_key_dict=user_api_key_dict, + # model, credentials, ...) from the registry: the same source the direct + # /vector_stores/{id}/search endpoint uses. Store-managed keys win on + # conflict so callers cannot override the store's provider or credentials. + managed_store: Final = resolved_stores.get(retrieval_config["vector_store_id"]) + store_data: Final = ( + await build_request_data_from_managed_vector_store(managed_store) + if managed_store is not None + else MappingProxyType({}) ) merged_retrieval_config: Final = { - **store_data, **retrieval_config, + **store_data, } # mutable-ok: litellm.aquery requires a plain dict payload # Add litellm data @@ -733,7 +747,12 @@ async def rag_query( proxy_config=proxy_config, ) - verbose_proxy_logger.debug("RAG Query - model: %s, retrieval_config: %s", model, merged_retrieval_config) + verbose_proxy_logger.debug( + "RAG Query - model: %s, vector_store_id: %s, custom_llm_provider: %s", + model, + retrieval_config["vector_store_id"], + merged_retrieval_config.get("custom_llm_provider"), + ) # Call query response: Final = await litellm.aquery( diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index a59d7a277cc..3fc6749f18a 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import ( Annotated, Any, # noqa: TID251 # jsonify_object in proxy/utils.py is annotated with a bare dict @@ -32,6 +34,41 @@ router: Final = APIRouter() ######################################################## +async def build_request_data_from_managed_vector_store( + vector_store: LiteLLM_ManagedVectorStore, +) -> Mapping[str, object]: + """ + Build request params (provider, credential ref, litellm_params) from an + already-resolved managed vector store. + + ``litellm_embedding_config`` is resolved here, at request-handling time, + instead of at row-creation time: the resolved api_key/api_base/api_version + lives only in the returned per-request mapping and is never persisted back + to the registry cache. Legacy rows that already carry a resolved + (cleartext) config skip the lookup and pass through unchanged. + """ + top_level: Final = MappingProxyType( + { + key: vector_store.get(key) + for key in ("custom_llm_provider", "litellm_credential_name") + if key in vector_store + } + ) + litellm_params: Final = vector_store.get("litellm_params") or MappingProxyType({}) + embedding_model: Final = litellm_params.get("litellm_embedding_model") + if not embedding_model or litellm_params.get("litellm_embedding_config"): + return MappingProxyType({**top_level, **litellm_params}) + + from litellm.proxy.proxy_server import prisma_client + + resolved_config: Final = await _resolve_embedding_config( + embedding_model=embedding_model, prisma_client=prisma_client + ) + if not resolved_config: + return MappingProxyType({**top_level, **litellm_params}) + return MappingProxyType({**top_level, **litellm_params, "litellm_embedding_config": resolved_config}) + + async def _update_request_data_with_litellm_managed_vector_store_registry( data: dict, vector_store_id: str, @@ -51,47 +88,14 @@ async def _update_request_data_with_litellm_managed_vector_store_registry( vector_store_to_run: Final[LiteLLM_ManagedVectorStore | None] = await get_litellm_managed_vector_store( vector_store_id=vector_store_id ) - if vector_store_to_run is not None: - if user_api_key_dict is not None: - await assert_user_can_access_vector_store( - vector_store=vector_store_to_run, - user_api_key_dict=user_api_key_dict, - ) - - if "custom_llm_provider" in vector_store_to_run: - data["custom_llm_provider"] = vector_store_to_run.get("custom_llm_provider") - - if "litellm_credential_name" in vector_store_to_run: - data["litellm_credential_name"] = vector_store_to_run.get("litellm_credential_name") - - if "litellm_params" in vector_store_to_run: - litellm_params = vector_store_to_run.get("litellm_params", {}) or {} - # Resolve ``litellm_embedding_config`` here, at request-handling - # time, instead of at row-creation time. The resolved - # ``api_key`` / ``api_base`` / ``api_version`` lives only in - # this per-request ``data`` dict and is never persisted. - # Legacy rows that already carry a resolved (cleartext) - # ``litellm_embedding_config`` skip the lookup and pass through - # unchanged so the embed call keeps working. - embedding_model: Final = litellm_params.get("litellm_embedding_model") - if embedding_model and not litellm_params.get("litellm_embedding_config"): - from litellm.proxy.proxy_server import prisma_client - - resolved_config: Final = await _resolve_embedding_config( - embedding_model=embedding_model, prisma_client=prisma_client - ) - if resolved_config: - # Build a fresh dict via spread instead of mutating - # ``litellm_params`` in place — the registry hands back - # a reference to its cached object, so an in-place - # update would persist the resolved cleartext into the - # in-memory cache for the lifetime of the process. - litellm_params = { - **litellm_params, - "litellm_embedding_config": resolved_config, - } - data.update(litellm_params) - return data + if vector_store_to_run is None: + return data + if user_api_key_dict is not None: + await assert_user_can_access_vector_store( + vector_store=vector_store_to_run, + user_api_key_dict=user_api_key_dict, + ) + return {**data, **(await build_request_data_from_managed_vector_store(vector_store_to_run))} @router.post( diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 183a03cc13c..244798ba05e 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -470,7 +470,7 @@ async def create_vector_store_in_db( # exposed every env-stored embedding-model credential on the # ``/vector_store/{new,info,update,list}`` responses. Keep the user's # raw ``litellm_embedding_model`` reference; resolution now happens in - # ``_update_request_data_with_litellm_managed_vector_store_registry`` + # ``build_request_data_from_managed_vector_store`` # at request-handling time so the cleartext config exists only in # per-request memory and never reaches the database. if litellm_params: @@ -864,7 +864,7 @@ async def update_vector_store( # embedding-config auto-resolve previously persisted cleartext # credentials into the row; resolution now happens at request- # handling time in - # ``_update_request_data_with_litellm_managed_vector_store_registry`` + # ``build_request_data_from_managed_vector_store`` # so this row only ever stores the user-supplied # ``litellm_embedding_model`` reference. if "litellm_params" in update_data: diff --git a/litellm/rag/main.py b/litellm/rag/main.py index bd6788b3a1b..94bfc305a6a 100644 --- a/litellm/rag/main.py +++ b/litellm/rag/main.py @@ -51,12 +51,19 @@ INGESTION_REGISTRY: Final[dict[str, type[BaseRAGIngestion]]] = { "vertex_ai": VertexAIRAGIngestion, } -# retrieval_config keys consumed by the query pipeline itself; everything else is -# forwarded to vector_stores.asearch as provider-specific params (e.g. -# aws_region_name, embedding_model, vector_bucket_name for S3 Vectors). -# `filters`/`retrieval_filter` are reserved for the explicit filter param. -_CONSUMED_RETRIEVAL_CONFIG_KEYS: Final = frozenset( - {"vector_store_id", "custom_llm_provider", "top_k", "filters", "retrieval_filter"} +# Only these retrieval_config keys are forwarded to vector_stores.asearch as +# provider-specific params. The explicit allowlist keeps caller-controlled +# connection overrides (api_base, api_key, ...) away from the search call, +# where they could redirect store credentials to an attacker-chosen host. +_FORWARDABLE_RETRIEVAL_CONFIG_KEYS: Final = frozenset( + { + "aws_region_name", + "vector_bucket_name", + "embedding_model", + "litellm_embedding_model", + "litellm_embedding_config", + "litellm_credential_name", + } ) @@ -233,10 +240,10 @@ async def _execute_query_pipeline( raise ValueError("No query found in messages for RAG query") # 2. Search vector store - # Forward provider-specific retrieval_config extras (region, embedding model, - # bucket, credentials refs, ...) to the search call; kwargs win on conflict. + # Forward allowlisted provider retrieval_config extras (region, embedding + # model, bucket, credential refs) to the search call; kwargs win on conflict. provider_search_params: Final = MappingProxyType( - {k: v for k, v in retrieval_config.items() if k not in _CONSUMED_RETRIEVAL_CONFIG_KEYS} + {k: v for k, v in retrieval_config.items() if k in _FORWARDABLE_RETRIEVAL_CONFIG_KEYS} ) forwarded_search_params: Final = MappingProxyType({**provider_search_params, **kwargs}) with _suppressed_sub_call_billing(): diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 5561ee1e6ae..342a4535b21 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -357,12 +357,9 @@ def test_rag_query_merges_managed_store_params(client_internal_user): "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", new_callable=AsyncMock, return_value=mock_response, - ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and stubs the access assert covered by auth tests - "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", - new=AsyncMock(), - ), patch( # test-quality-ok: stubs the direct-endpoint access assert covered by auth tests - "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", - new=AsyncMock(), + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and grants access so real store resolution runs + "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", + new=AsyncMock(return_value=True), ): response = client_internal_user.post( "/v1/rag/query", @@ -383,8 +380,8 @@ def test_rag_query_merges_managed_store_params(client_internal_user): assert forwarded_config["vector_bucket_name"] == "bkt" -def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): - """User-supplied retrieval_config keys must win over registry values.""" +def test_rag_query_store_params_win_over_user_retrieval_config(client_internal_user): + """Registry values must win over user-supplied retrieval_config keys so callers cannot override store credentials.""" import litellm from litellm.types.utils import ModelResponse @@ -406,12 +403,9 @@ def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", new_callable=AsyncMock, return_value=mock_response, - ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and stubs the access assert covered by auth tests - "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", - new=AsyncMock(), - ), patch( # test-quality-ok: stubs the direct-endpoint access assert covered by auth tests - "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", - new=AsyncMock(), + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and grants access so real store resolution runs + "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", + new=AsyncMock(return_value=True), ): response = client_internal_user.post( "/v1/rag/query", @@ -424,7 +418,9 @@ def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): assert response.status_code == 200, response.json() forwarded_config = mock_aquery.await_args.kwargs["retrieval_config"] - assert forwarded_config["aws_region_name"] == "us-east-1" + assert forwarded_config["aws_region_name"] == "eu-west-1" + + EICAR = r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" INGEST_REQUEST = '{"ingest_options":{"vector_store":{"custom_llm_provider":"openai"}}}' diff --git a/tests/test_litellm/rag/test_main.py b/tests/test_litellm/rag/test_main.py index fdcdf342eea..51d03544910 100644 --- a/tests/test_litellm/rag/test_main.py +++ b/tests/test_litellm/rag/test_main.py @@ -349,6 +349,45 @@ async def test_aquery_minimal_retrieval_config_forwards_no_extras(): assert not (leaked & set(search_kwargs.keys())) +@pytest.mark.asyncio +async def test_aquery_does_not_forward_connection_override_keys_to_search(): + """ + Only allowlisted retrieval_config keys may reach the vector store search + call. Caller-controlled connection overrides (api_base, api_key, arbitrary + extras) must be dropped, otherwise a caller could redirect store + credentials to an attacker-chosen host. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): # test-quality-ok: asearch is the boundary the forwarding contract under test targets + await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={ + "vector_store_id": "bkt:idx", + "custom_llm_provider": "s3_vectors", + "aws_region_name": "eu-west-1", + "api_base": "https://attacker.example.com", + "api_key": "attacker-key", + "arbitrary_extra": "nope", + }, + mock_response="hi", + ) + + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["aws_region_name"] == "eu-west-1" + blocked = {"api_base", "api_key", "arbitrary_extra"} + assert not (blocked & set(search_kwargs.keys())) + + def test_rag_call_types_are_registered(): """ query/aquery/ingest/aingest are @client-decorated entry points, so their From 8b5ae3da9d49e08c4f6c8ca22d6f59c64c6bfae5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:15:51 -0700 Subject: [PATCH 09/67] test(vector_stores): package the suite dir to avoid test_main basename collision --- tests/test_litellm/vector_stores/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/test_litellm/vector_stores/__init__.py diff --git a/tests/test_litellm/vector_stores/__init__.py b/tests/test_litellm/vector_stores/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 8b0441a628c01f0cd6caa10176ae887c06d75fa7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:29:25 -0700 Subject: [PATCH 10/67] fix(vector_stores): block caller-supplied embedding selection params on query surfaces --- litellm/proxy/rag_endpoints/endpoints.py | 2 ++ .../proxy/vector_store_endpoints/endpoints.py | 24 ++++++++++++++ .../proxy/rag_endpoints/test_rag_endpoints.py | 24 ++++++++++++++ .../test_vector_store_endpoints.py | 32 +++++++++++++++++++ 4 files changed, 82 insertions(+) diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index d2c7d6f93ee..0ab7d99e4e4 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -42,6 +42,7 @@ from litellm.proxy.rag_endpoints.upload_security import ( ) from litellm.proxy.vector_store_endpoints.endpoints import ( build_request_data_from_managed_vector_store, + reject_caller_embedding_selection_params, ) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, @@ -716,6 +717,7 @@ async def rag_query( status_code=400, detail={"error": "retrieval_config must contain 'vector_store_id'"}, ) + reject_caller_embedding_selection_params(payload=retrieval_config, source="retrieval_config") resolved_stores: Final = await _authorize_nested_vector_store_ids( payload=retrieval_config, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 3fc6749f18a..7d64e648e08 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -29,6 +29,29 @@ from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry router: Final = APIRouter() + +BLOCKED_QUERY_EMBEDDING_SELECTION_PARAMS: Final = frozenset( + { + "embedding_model", + "litellm_embedding_model", + "litellm_embedding_config", + "litellm_credential_name", + } +) + + +def reject_caller_embedding_selection_params(payload: Mapping[str, object], source: str) -> None: + blocked: Final = sorted(BLOCKED_QUERY_EMBEDDING_SELECTION_PARAMS & payload.keys()) + if blocked: + raise HTTPException( + status_code=400, + detail={ + "error": f"'{blocked[0]}' cannot be set in {source}. " + "Embedding configuration comes from the vector store's server-side registration." + }, + ) + + ######################################################## # OpenAI Compatible Endpoints ######################################################## @@ -134,6 +157,7 @@ async def vector_store_search( ) data = await _read_request_body(request=request) + reject_caller_embedding_selection_params(payload=data, source="the search request body") data["vector_store_id"] = vector_store_id # Check for legacy vector store registry (non-managed vector stores) diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 342a4535b21..0085b6ebd36 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -421,6 +421,30 @@ def test_rag_query_store_params_win_over_user_retrieval_config(client_internal_u assert forwarded_config["aws_region_name"] == "eu-west-1" +@pytest.mark.parametrize( + "blocked_key", + ["embedding_model", "litellm_embedding_model", "litellm_embedding_config", "litellm_credential_name"], +) +def test_rag_query_rejects_caller_embedding_selection_params(client_internal_user, blocked_key): + """ + Regression: a caller must not pick the embedding model or credential used at + search time. Those resolve through the Router with the proxy's credentials, + bypassing the key's model permissions, so they may only come from the + managed store's server-side registration. + """ + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store", blocked_key: "attacker-choice"}, + }, + ) + + assert response.status_code == 400, response.json() + assert blocked_key in str(response.json()) + + EICAR = r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" INGEST_REQUEST = '{"ingest_options":{"vector_store":{"custom_llm_provider":"openai"}}}' diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index eae6f90863a..45a0221c8a6 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -3158,3 +3158,35 @@ class TestAzureAIAnalyzeNamedIndexClassification: user_api_key_dict=self._team_member("analyze", ["read"]), ) assert result is True + + +@pytest.mark.parametrize( + "blocked_key", + ["embedding_model", "litellm_embedding_model", "litellm_embedding_config", "litellm_credential_name"], +) +def test_vector_store_search_rejects_caller_embedding_selection_params(blocked_key): + """ + Regression: the search request body must not pick the embedding model or + credential used to embed the query. Those resolve through the Router with + the proxy's credentials, bypassing the key's model permissions, so they may + only come from the managed store's server-side registration. + """ + from fastapi.testclient import TestClient + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + + mock_auth = UserAPIKeyAuth(user_id="test_internal_user", user_role=LitellmUserRoles.INTERNAL_USER.value) + original_overrides = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: mock_auth + try: + client = TestClient(app) + response = client.post( + "/v1/vector_stores/s3-store/search", + json={"query": "hello", blocked_key: "attacker-choice"}, + ) + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 400, response.json() + assert blocked_key in str(response.json()) From 1eea8e283157d3c92be36749b2713607aebc9786 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:54:34 -0700 Subject: [PATCH 11/67] fix(deps): raise the tornado floor to 6.5.8 for GHSA-8423-8fgw-73vq and GHSA-wwv5-g3v4-889x --- pyproject.toml | 2 +- uv.lock | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2866e27e84c..d0f14722acd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -292,7 +292,7 @@ exclude = [ [tool.uv] constraint-dependencies = [ - "tornado>=6.5.6", + "tornado>=6.5.8", "aiohttp>=3.14.2,<4.0", "packaging>=24.0", "soupsieve>=2.8.4", diff --git a/uv.lock b/uv.lock index 27be919eea1..8bac024d49e 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-29T17:58:57.633306Z" +exclude-newer = "2026-08-29T20:52:40.322465Z" exclude-newer-span = "P3D" [manifest] @@ -25,7 +25,7 @@ constraints = [ { name = "packaging", specifier = ">=24.0" }, { name = "setuptools", specifier = ">=83.0.0" }, { name = "soupsieve", specifier = ">=2.8.4" }, - { name = "tornado", specifier = ">=6.5.6" }, + { name = "tornado", specifier = ">=6.5.8" }, ] overrides = [ { name = "cryptography", specifier = ">=50.0.0,<51.0" }, @@ -9441,19 +9441,19 @@ wheels = [ [[package]] name = "tornado" -version = "6.5.7" +version = "6.5.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/64/24/95ec527ad67b76d59299e5465b3935d05e4294b7e0290a3924b7487df30b/tornado-6.5.7.tar.gz", hash = "sha256:66c513a76cda70d53907bc27cf1447557699c2e95aa48ba27a442ff61c3ddfc2", size = 519252, upload-time = "2026-06-08T17:34:51.232Z" } +sdist = { url = "https://files.pythonhosted.org/packages/10/d3/343e5bb989d6515b1646cf3d40135d73f3d5e45339bded401b56cdac24dd/tornado-6.5.8.tar.gz", hash = "sha256:9452e1b208a8bd771e2cb1f2ff564985b9b214bdebbe622793e1799e0a6bd23f", size = 520493, upload-time = "2026-08-07T02:12:42.971Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/dc/c7043cab6fed8ae159fc1923ce829ada35c4dbd797d408a43858ffaf9639/tornado-6.5.7-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:148b2eb15c2c765a50796172c1e499649b35f30d2e3c3d3e15913cfa56bfb163", size = 448543, upload-time = "2026-06-08T17:34:38.052Z" }, - { url = "https://files.pythonhosted.org/packages/92/4f/090b1431e5a43df696feceffc268c5383cc079ecb5f08ce58f917109aafe/tornado-6.5.7-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9da38de27f1da3b78a966f0dae12b5a1ea9afe72ca805d84ff06508272ddf100", size = 446707, upload-time = "2026-06-08T17:34:39.594Z" }, - { url = "https://files.pythonhosted.org/packages/37/d8/ef374952fd5da67d4463122c2b8e5a96536ec10b4b339254c6dcde81d01c/tornado-6.5.7-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8d759e71906ee783f8867b93bf26a265743da4c1e2f4a018464c1ba019862972", size = 449774, upload-time = "2026-06-08T17:34:41.204Z" }, - { url = "https://files.pythonhosted.org/packages/35/37/d434c73f4c6e014b745b9b37085f34f40c022f007efff3d7fe65991899f3/tornado-6.5.7-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a46347a18f23fb92b396beebe0fb78f61dda0cc302445202c16203d8a18848b", size = 450745, upload-time = "2026-06-08T17:34:42.531Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2b/56b9aff361d7f1ab728a805ec7d7ea835f8807afa9f5cc690ea0e630efb9/tornado-6.5.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7778b30bef919231265e91c69963ce0f49a1e9c07ac900bbe75b19ce2575ba92", size = 450578, upload-time = "2026-06-08T17:34:43.787Z" }, - { url = "https://files.pythonhosted.org/packages/02/30/a7444fb23aa76860a14198fab96ac79f1866b0a6e19e26c4381b0938e50f/tornado-6.5.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e726f0c75da7726eec023aa62751ff8878bd2737e34fbdd33b1ae5897d2200f5", size = 449985, upload-time = "2026-06-08T17:34:45.326Z" }, - { url = "https://files.pythonhosted.org/packages/5c/42/5f0e56c01e8d9d36f4e23f367b85ae6cae0c1ecddd5e6977d8388ad27488/tornado-6.5.7-cp39-abi3-win32.whl", hash = "sha256:f8de3bf12d3efdd0cbe7c8887868198f8a91415e3f29fcf258d9b8eb7b1d9ae4", size = 451047, upload-time = "2026-06-08T17:34:46.784Z" }, - { url = "https://files.pythonhosted.org/packages/c9/a4/b393076ffb21b469eec5b328a0534cf03a3b90bfc6b1f09507cdd075d938/tornado-6.5.7-cp39-abi3-win_amd64.whl", hash = "sha256:de942f843533a039ef9fa3d9c88c7cd8a7c94553fb5ad0154270989b3d99a2c4", size = 451485, upload-time = "2026-06-08T17:34:48.248Z" }, - { url = "https://files.pythonhosted.org/packages/71/2e/7b1c769803121b809112cf9a00681c472eae1d80e32d7ec0e0bd61d0d0e1/tornado-6.5.7-cp39-abi3-win_arm64.whl", hash = "sha256:ff934fce95643af5f11efdae618eaa73d469dc588641e5c8d19295a0c65c4796", size = 450506, upload-time = "2026-06-08T17:34:49.702Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d5/007086fd8df5489338e204f65adce33fd4f21a4999dbb2b9cff2f897b5f4/tornado-6.5.8-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:cc6aa787d7cfab7c3d35189dc7a56fbd2399a569624c730c6b55b3d6531d0403", size = 449487, upload-time = "2026-08-07T02:12:28.682Z" }, + { url = "https://files.pythonhosted.org/packages/70/c8/5a24a99495903f594f6a199dd7beead1cbc0a13e2cb9102727bcaaf2a997/tornado-6.5.8-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9715b5eb79735b2bcd454ce216a9275b7c0470e64ea1bf5742f78b2f72b26eeb", size = 447649, upload-time = "2026-08-07T02:12:30.306Z" }, + { url = "https://files.pythonhosted.org/packages/6e/de/f2e733f386b85962d1b1dc82cd63d169b5b4580062b35397eac9244a41fe/tornado-6.5.8-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:547d63f450d570c14fe0e8db2cfb14c9bbd1c2503b4a6612586267955aa47b58", size = 450707, upload-time = "2026-08-07T02:12:31.95Z" }, + { url = "https://files.pythonhosted.org/packages/0b/94/20efeee9a01c141e9ac47c397f81679dfda24b32768fc4fff24e76d36c2c/tornado-6.5.8-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e2360a0ffbe145eca8af0b19cb7203d79b1a98dd4cccdd6b368f6f49c2e3808", size = 451677, upload-time = "2026-08-07T02:12:33.512Z" }, + { url = "https://files.pythonhosted.org/packages/42/ec/a96ccb8ccf0de2b7bc2c5fa1608a4803735018242e90c4882365a9fd418f/tornado-6.5.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5d242290bdf7ab3151bc1065fdd75c0dcc21cbc7b49f22a4c56329c2d6566d22", size = 451510, upload-time = "2026-08-07T02:12:35.346Z" }, + { url = "https://files.pythonhosted.org/packages/29/b5/93185859245ad3f00e62175f29607346788b696369347f0146e0421286bb/tornado-6.5.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7b94ff0e128fe0542f3bd331fb44d06260fc4ac16881545159f34ef08aad4195", size = 450917, upload-time = "2026-08-07T02:12:36.963Z" }, + { url = "https://files.pythonhosted.org/packages/97/cf/fe33cf062834487d34d1559746a4a12521033c22645b6d74d4bca702e018/tornado-6.5.8-cp39-abi3-win32.whl", hash = "sha256:67832909c4779c64942380cb5f044a5c6163d00831472d80e25e115de9917836", size = 451952, upload-time = "2026-08-07T02:12:38.512Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e1/468ad54333e92ccb62627e62cb88e5fc14a2171daa67ed47b1b8542d5b86/tornado-6.5.8-cp39-abi3-win_amd64.whl", hash = "sha256:11881db6b7c168494be2c2d12e65931451bdf7ee718535418ae1d8855dd5a0ee", size = 452391, upload-time = "2026-08-07T02:12:39.971Z" }, + { url = "https://files.pythonhosted.org/packages/ad/3e/cd5e4f06e34cde33b8ef66cf36aa2b5ad46354cc1af7d2136bbe365fee1d/tornado-6.5.8-cp39-abi3-win_arm64.whl", hash = "sha256:68a7468c7e289f8514d7d664101753903217eff1bb6822c6b5994a0b5f5bcb26", size = 451411, upload-time = "2026-08-07T02:12:41.469Z" }, ] [[package]] From 386946353ac425c6cbcac28368846dc2ab413bc2 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 14:39:15 -0700 Subject: [PATCH 12/67] fix(vertex): avoid duplicate DeepSeek OCR model namespace --- .../vertex_ai/ocr/deepseek_transformation.py | 3 ++- tests/ocr_tests/test_ocr_vertex_ai.py | 20 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py index 2603552152d..b57a87c3325 100644 --- a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py +++ b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py @@ -177,8 +177,9 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig): content_item = {"type": "image_url", "image_url": document_url} # Build DeepSeek OCR request + provider_model: Final = model if model.startswith("deepseek-ai/") else f"deepseek-ai/{model}" data: Final = { - "model": "deepseek-ai/" + model, + "model": provider_model, "messages": [{"role": "user", "content": [content_item]}], } diff --git a/tests/ocr_tests/test_ocr_vertex_ai.py b/tests/ocr_tests/test_ocr_vertex_ai.py index 1ba5b9d0883..1842eb063a5 100644 --- a/tests/ocr_tests/test_ocr_vertex_ai.py +++ b/tests/ocr_tests/test_ocr_vertex_ai.py @@ -5,9 +5,11 @@ Note: Vertex AI OCR automatically converts URLs to base64 data URIs since the Vertex AI endpoint doesn't have internet access. """ -import os import json +import os import tempfile +from typing import Final + import pytest from base_ocr_unit_tests import BaseOCRTest @@ -139,3 +141,19 @@ def test_vertex_ai_ocr_routing(): assert isinstance( deepseek_variant, VertexAIDeepSeekOCRConfig ), "DeepSeek variant should route to VertexAIDeepSeekOCRConfig" + + +@pytest.mark.parametrize("model", ("deepseek-ocr-maas", "deepseek-ai/deepseek-ocr-maas")) +def test_deepseek_request_uses_single_provider_namespace(model: str) -> None: + from litellm.llms.vertex_ai.ocr.deepseek_transformation import ( + VertexAIDeepSeekOCRConfig, + ) + + request: Final = VertexAIDeepSeekOCRConfig().transform_ocr_request( + model=model, + document={"type": "image_url", "image_url": "data:image/png;base64,AA=="}, + optional_params={}, + headers={}, + ) + + assert request.data["model"] == "deepseek-ai/deepseek-ocr-maas" From d59fcda8af69f5545a8e7c29b29d12362e2bffad Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:47:59 -0700 Subject: [PATCH 13/67] fix(rerank): adopt declared authenticating providers in arerank instead of resolving them get_llm_provider runs the OAuth device flow for github_copilot and chatgpt, so calling it on the event loop before the executor dispatch let an authenticated caller block the loop for the length of the polling window. Adopt the declared provider via declared_authenticating_provider, matching the metadata callers in utils.py, and only resolve for everything else. --- litellm/rerank_api/main.py | 19 +++++++++---- tests/test_litellm/rerank_api/test_main.py | 32 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index 597d1cfb863..37ca989b8d3 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -6,6 +6,7 @@ from typing import Any, Final, Literal import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler @@ -43,16 +44,22 @@ async def arerank( """ Async: Reranks a list of documents based on their relevance to the query """ - _custom_llm_provider: str | None = None # rebind-ok: set by the get_llm_provider unpack; read in the except + _custom_llm_provider: str | None = ( + None # rebind-ok: set by the declared-provider guard or the get_llm_provider unpack; read in the except + ) try: loop: Final = asyncio.get_event_loop() kwargs["arerank"] = True - _, _custom_llm_provider, _, _ = litellm.get_llm_provider( # rebind-ok: see pre-declaration above - model=model, - custom_llm_provider=custom_llm_provider, - api_base=kwargs.get("api_base", None), - ) + declared_provider: Final = declared_authenticating_provider(model, custom_llm_provider) + if declared_provider is not None: + _custom_llm_provider = declared_provider # rebind-ok: see pre-declaration above + else: + _, _custom_llm_provider, _, _ = litellm.get_llm_provider( # rebind-ok: see pre-declaration above + model=model, + custom_llm_provider=custom_llm_provider, + api_base=kwargs.get("api_base", None), + ) func: Final = partial( rerank, diff --git a/tests/test_litellm/rerank_api/test_main.py b/tests/test_litellm/rerank_api/test_main.py index 62149c742d6..2b6cfeda2c2 100644 --- a/tests/test_litellm/rerank_api/test_main.py +++ b/tests/test_litellm/rerank_api/test_main.py @@ -172,6 +172,38 @@ async def test_arerank_error_is_mapped_to_litellm_exception(respx_mock: respx.Mo assert "None - " not in str(exc_info.value) +@pytest.mark.asyncio +async def test_arerank_declared_authenticating_provider_skips_resolution(monkeypatch): + """Regression for the event-loop hazard in arerank's provider pre-resolution: + get_llm_provider runs the blocking OAuth device flow for github_copilot/chatgpt, + so arerank must adopt the declared provider instead of resolving it, while the + except path still maps with that declared provider.""" + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + resolution_calls = [] + + def record_resolution(*args, **kwargs): + resolution_calls.append((args, kwargs)) + return "gpt-4o", "github_copilot", None, None + + def rerank_raises_provider_error(*args, **kwargs): + raise BaseLLMException(status_code=401, message='{"error":"bad key"}') + + monkeypatch.setattr(litellm, "get_llm_provider", record_resolution) + monkeypatch.setattr("litellm.rerank_api.main.rerank", rerank_raises_provider_error) + + with pytest.raises(litellm.AuthenticationError) as exc_info: + await litellm.arerank( + model="github_copilot/gpt-4o", + query=MARKER_QUERY, + documents=[MARKER_DOC], + ) + + assert resolution_calls == [] + assert "Github_copilotException" in str(exc_info.value) + assert "None - " not in str(exc_info.value) + + @pytest.mark.asyncio async def test_together_rerank_async_honors_env_api_base(respx_mock: respx.MockRouter, monkeypatch): """Regression: TOGETHER_AI_API_BASE was honored by chat but ignored by rerank.""" From fcd9052179f039f075b3e25a8c1aec8657fc98a0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:10:52 -0700 Subject: [PATCH 14/67] feat(proxy): honor model_info.display_name in the Anthropic-shaped /v1/models listing --- litellm/llms/anthropic/common_utils.py | 17 ++-- .../proxy/common_utils/model_listing_utils.py | 25 +++++- litellm/proxy/proxy_server.py | 21 +++-- litellm/router.py | 20 +++++ .../proxy/proxy_server/test_routes_models.py | 78 +++++++++++++++++++ .../test_team_model_name_translation.py | 26 ++++++- tests/test_litellm/test_router.py | 65 ++++++++++++++++ 7 files changed, 240 insertions(+), 12 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index c60ebd844ba..d23690976ad 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1378,31 +1378,38 @@ def process_anthropic_headers(headers: httpx.Headers | dict) -> dict: return additional_headers -def _anthropic_model_entry(model: ModelInfoResponse, created_at: str) -> Mapping[str, object]: +def _anthropic_model_entry( + model: ModelInfoResponse, created_at: str, display_names: Mapping[str, str] +) -> Mapping[str, object]: return { # mutable-ok: JSON response body, serialized by the route and never mutated "type": "model", "id": model["id"], - "display_name": model["id"], + "display_name": display_names.get(model["id"], model["id"]), "created_at": created_at, "max_input_tokens": model.get("max_input_tokens"), "max_tokens": model.get("max_output_tokens"), } -def create_anthropic_model_list_response(models: Sequence[ModelInfoResponse]) -> Mapping[str, object]: +def create_anthropic_model_list_response( + models: Sequence[ModelInfoResponse], + display_names: Mapping[str, str] = MappingProxyType({}), +) -> Mapping[str, object]: """Build the Anthropic-native /v1/models envelope. Clients that send an anthropic-version header parse the Anthropic Models API shape (type/display_name/created_at plus has_more/first_id/last_id) and filter the list themselves, so every model is returned here. The token limits carry over from the OpenAI-shaped listing, named as the Messages API names them, and - are always present because the vendor shape declares them nullable, not optional + are always present because the vendor shape declares them nullable, not optional. + display_names maps a listed model id to a configured human-readable name; ids + without an entry fall back to the id itself, matching the vendor behavior """ created_at: Final = ( datetime.fromtimestamp(DEFAULT_MODEL_CREATED_AT_TIME, tz=timezone.utc).isoformat().replace("+00:00", "Z") ) data: Final = [ # mutable-ok: JSON response body, serialized by the route and never mutated - _anthropic_model_entry(model, created_at) for model in models + _anthropic_model_entry(model, created_at, display_names) for model in models ] return { # mutable-ok: JSON response body, serialized by the route and never mutated "data": data, diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index 9fd24162f7e..213a697b3dd 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -10,13 +10,36 @@ legacy internal names with `general_settings.use_team_public_model_name: false`. from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast if TYPE_CHECKING: from litellm.router import Router +def configured_display_names( + entries: Sequence[tuple[str, str]], + llm_router: Router | None, +) -> Mapping[str, str]: + """response_id -> configured `model_info.display_name` for the listing entries + that have one. + + Metadata is looked up by each entry's internal lookup id (so team-scoped rows + resolve), while the returned map is keyed by the public response id the + Anthropic-shaped listing is built from. Entries without a configured name are + omitted so the listing falls back to the id itself. + """ + if llm_router is None: + return MappingProxyType({}) + resolved: Final = ( + (response_id, llm_router.get_configured_display_name(lookup_id)) for response_id, lookup_id in entries + ) + return MappingProxyType( + {response_id: display_name for response_id, display_name in resolved if display_name is not None} + ) + + class TeamModelNameTranslator: """Translates internal team routing keys to their public names for the model listing/retrieve responses. Stateless; the live router and general_settings diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2c600667283..4f172ca29b9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -351,7 +351,10 @@ from litellm.proxy.common_utils.load_config_utils import ( get_file_contents_from_s3, ) from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations -from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator +from litellm.proxy.common_utils.model_listing_utils import ( + TeamModelNameTranslator, + configured_display_names, +) from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, ) @@ -10193,7 +10196,8 @@ async def model_list( # The internal routing key drives the metadata/fallback lookup, while the # public name is what the client sees as the model id. model_data = [] - for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings): + admin_entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings) + for response_id, lookup_id in admin_entries: model_info = create_model_info_response( model_id=lookup_id, provider="openai", @@ -10206,7 +10210,10 @@ async def model_list( if wants_anthropic_format: admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above - return create_anthropic_model_list_response(admin_listing) + return create_anthropic_model_list_response( + admin_listing, + display_names=configured_display_names(admin_entries, llm_router), + ) return dict( data=model_data, @@ -10237,7 +10244,8 @@ async def model_list( # The internal routing key drives the metadata/fallback lookup, while the # public name is what the client sees as the model id. model_data = [] - for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings): + entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings) + for response_id, lookup_id in entries: model_info = create_model_info_response( model_id=lookup_id, provider="openai", @@ -10250,7 +10258,10 @@ async def model_list( if wants_anthropic_format: listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above - return create_anthropic_model_list_response(listing) + return create_anthropic_model_list_response( + listing, + display_names=configured_display_names(entries, llm_router), + ) return dict( data=model_data, diff --git a/litellm/router.py b/litellm/router.py index 471a1116f44..6245f8a6a03 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9607,6 +9607,26 @@ class Router: coerce_token_limit(model_info.get("max_output_tokens")), ) + def get_configured_display_name(self, model_name: str) -> "str | None": + """ + Return the display_name explicitly configured in a concrete deployment's + model_info for model_name, via O(1) index lookup. + + Returns None for wildcard-expanded or unknown names, and treats a + non-string or empty configured value as absent rather than failing the + listing. Like get_configured_token_limits, this never triggers pattern + matching or deep copies, so it is safe to call per listed model on the + /v1/models hot path. + """ + deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return None + + display_name: Final = deployment.model_info.get("display_name") + if isinstance(display_name, str) and display_name.strip(): + return display_name + return None + def get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None ) -> dict[str, Any] | None: diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_models.py b/tests/test_litellm/proxy/proxy_server/test_routes_models.py index 2b126b1ea95..bc6106a06f8 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_models.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_models.py @@ -45,6 +45,7 @@ def patched_models(monkeypatch): deployment = MagicMock() deployment.litellm_params.model = "gpt-4" router.get_deployment_by_model_group_name = MagicMock(return_value=deployment) + router.get_configured_display_name = MagicMock(return_value=None) monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) @@ -187,6 +188,83 @@ def test_anthropic_format_carries_router_configured_token_limits(client, auth_as assert (claude["max_input_tokens"], claude["max_tokens"]) == (500000, 4096) +@pytest.mark.parametrize("path", ["/v1/models", "/models"]) +def test_anthropic_format_uses_configured_display_name(client, auth_as, patched_models, path): + """A deployment's ``model_info.display_name`` becomes the Anthropic-native + ``display_name`` so Claude Code's picker shows a clean name while the id keeps + routing; models without one keep the id fallback, and the OpenAI-shaped + listing carries no display_name either way.""" + + def _configured(model_name): + return "Kimi K3" if model_name == "gpt-4" else None + + patched_models.get_configured_display_name = MagicMock(side_effect=_configured) + + with auth_as(): + anthropic_response = client.get(path, headers={"anthropic-version": "2023-06-01"}) + openai_response = client.get(path) + + assert anthropic_response.status_code == 200 + gpt_4, claude = anthropic_response.json()["data"] + assert (gpt_4["id"], gpt_4["display_name"]) == ("gpt-4", "Kimi K3") + assert (claude["id"], claude["display_name"]) == ("claude-sonnet", "claude-sonnet") + + assert openai_response.status_code == 200 + openai_models = openai_response.json()["data"] + assert [m["id"] for m in openai_models] == ["gpt-4", "claude-sonnet"] + assert all("display_name" not in m for m in openai_models) + + +@pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) +def test_anthropic_display_name_resolved_via_internal_team_key( + client, auth_as, patched_models, monkeypatch, params +): + """For a team-scoped row the configured display name must be looked up by the + internal routing key while the entry itself is keyed by the public name, so + the clean name lands on the id the client actually sees.""" + from litellm.proxy import utils as proxy_utils + from litellm.proxy.auth import model_checks + + internal_name = "model_name_team-1_c0ffee" + + patched_models.get_model_list = MagicMock( + return_value=[ + { + "model_name": internal_name, + "model_info": { + "team_id": "team-1", + "team_public_model_name": "gpt-4-team", + }, + } + ] + ) + patched_models.get_model_names = MagicMock(return_value=[internal_name]) + patched_models.get_configured_display_name = MagicMock( + side_effect=lambda model_name: "Team GPT" if model_name == internal_name else None + ) + + async def _fake_get_available_models_for_user(**kwargs): + return [internal_name] + + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + _fake_get_available_models_for_user, + ) + monkeypatch.setattr( + model_checks, "get_complete_model_list", lambda **kwargs: [internal_name] + ) + + with auth_as(): + response = client.get( + "/v1/models", params=params, headers={"anthropic-version": "2023-06-01"} + ) + + assert response.status_code == 200 + (entry,) = response.json()["data"] + assert (entry["id"], entry["display_name"]) == ("gpt-4-team", "Team GPT") + + @pytest.mark.parametrize("path", ["/v1/models", "/models"]) def test_get_models_invalid_scope_returns_400(client, auth_as, patched_models, path): """Pins: ``GET /v1/models``, ``GET /models`` (error path: invalid scope).""" diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index aa35fd64f18..0fb9b1a6d88 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -19,7 +19,10 @@ from litellm.proxy._types import ( LitellmUserRoles, UserAPIKeyAuth, ) -from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator +from litellm.proxy.common_utils.model_listing_utils import ( + TeamModelNameTranslator, + configured_display_names, +) from litellm.proxy.proxy_server import ( _get_proxy_model_info, _translate_model_name_for_response, @@ -1391,6 +1394,27 @@ def test_resolve_public_name_respects_legacy_flag(): ) +def test_configured_display_names_keyed_by_response_id(): + """The map is keyed by the public response id while the router lookup uses + the internal routing key, and entries without a configured name are omitted.""" + router = MagicMock() + router.get_configured_display_name = MagicMock( + side_effect=lambda model_name: "Team Sonnet" if model_name == "model_name_team-abc-123_4a6b8" else None + ) + + assert configured_display_names( + entries=[ + ("team-claude-sonnet", "model_name_team-abc-123_4a6b8"), + ("gpt-4o", "gpt-4o"), + ], + llm_router=router, + ) == {"team-claude-sonnet": "Team Sonnet"} + + +def test_configured_display_names_empty_without_router(): + assert configured_display_names(entries=[("gpt-4o", "gpt-4o")], llm_router=None) == {} + + @pytest.mark.asyncio async def test_retrieve_model_by_public_name_returns_200(monkeypatch): """Regression: `GET /v1/models/{public_name}` must NOT 404. The listing diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 44c1cdbff06..f4ea9b03a80 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7271,6 +7271,71 @@ def test_get_configured_token_limits_coerces_numeric_strings(): assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000) +def test_get_configured_display_name_reads_deployment_model_info(): + router = litellm.Router( + model_list=[ + { + "model_name": "Kimi K3-claude-compatible", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"display_name": "Kimi K3"}, + } + ] + ) + + assert router.get_configured_display_name("Kimi K3-claude-compatible") == "Kimi K3" + + +def test_get_configured_display_name_returns_none_for_unset_or_unknown(): + router = litellm.Router( + model_list=[ + { + "model_name": "no-display-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + } + ] + ) + + assert router.get_configured_display_name("no-display-model") is None + assert router.get_configured_display_name("not-a-real-model") is None + + +def test_get_configured_display_name_skips_wildcard_pattern_matching(): + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock/*", + "litellm_params": {"model": "bedrock/*"}, + "model_info": {"display_name": "Bedrock"}, + } + ] + ) + + with patch.object( + router.pattern_router, "route", side_effect=AssertionError("pattern route called") + ): + assert ( + router.get_configured_display_name("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") + is None + ) + + +def test_get_configured_display_name_treats_malformed_values_as_absent(): + malformed = ["", " ", 12345, ["Kimi K3"], {"name": "Kimi K3"}, True] + router = litellm.Router( + model_list=[ + { + "model_name": f"bad-display-{i}", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"display_name": bad}, + } + for i, bad in enumerate(malformed) + ] + ) + + for i in range(len(malformed)): + assert router.get_configured_display_name(f"bad-display-{i}") is None + + @pytest.mark.asyncio async def test_acreate_batch_disable_fallbacks_surfaces_owning_provider_error(): router = litellm.Router( From f49a3e15a8b936c94e5d050017f6887e8d34e997 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:12:07 -0700 Subject: [PATCH 15/67] test(e2e): read JUnit properties off the real collected pytest Item tests/e2e/test_junit_properties.py fed a hand-rolled FakeItem to result_properties and attach_result_properties, both typed pytest.Item, so uv run basedpyright tests/e2e reported 3 reportArgumentType errors on litellm_internal_staging and every make check that scopes a litellm/ or tests/e2e/ Python file failed. Each test now looks up its own collected Item in request.session.items and applies the covers marker at run time through request.applymarker, so the coverage registry's collect-only pass never sees the test ids and the production functions keep their pytest.Item signatures. No casts, no ignores. Resolves LIT-6669 --- tests/e2e/test_junit_properties.py | 45 ++++++++++-------------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/tests/e2e/test_junit_properties.py b/tests/e2e/test_junit_properties.py index c0596177cc1..02c1413c840 100644 --- a/tests/e2e/test_junit_properties.py +++ b/tests/e2e/test_junit_properties.py @@ -24,25 +24,10 @@ from junit_properties import ( ) -class FakeMarker: - def __init__(self, name: str, *args: object) -> None: - self.name = name - self.args = args - - -class FakeItem: - """The three attributes junit_properties reads off a pytest Item.""" - - def __init__( - self, nodeid: str, location: tuple[str, int | None, str], markers: tuple[FakeMarker, ...] = () - ) -> None: - self.nodeid = nodeid - self.location = location - self.user_properties: list[tuple[str, str]] = [] - self._markers = markers - - def iter_markers(self, name: str): - return (marker for marker in self._markers if marker.name == name) +def collected_item(request: pytest.FixtureRequest, name: str) -> pytest.Item: + """The Item pytest collected for test ``name`` in this file: the real nodeid, + location and marker machinery the collection hook reads, as pytest built it.""" + return next(item for item in request.session.items if item.path == request.path and item.name == name) def repo_root() -> Path | None: @@ -109,22 +94,22 @@ class TestSourceFromLocation: class TestResultProperties: - def test_every_test_carries_package_covers_and_source(self) -> None: - item = FakeItem( - "logging/test_x.py::TestFoo::test_bar", - ("logging/test_x.py", 40, "TestFoo.test_bar"), - (FakeMarker("covers", "LOG-1", "LOG-2"),), - ) - assert result_properties(item) == ( - ("package", "logging"), + def test_every_test_carries_package_covers_and_source(self, request: pytest.FixtureRequest) -> None: + """Read off this test's own collected Item, so the nodeid and location are + whatever pytest reports for the launch shape in use, and the marker is added + at run time so the coverage registry's collect-only pass never sees it.""" + test = type(self).test_every_test_carries_package_covers_and_source + request.applymarker(pytest.mark.covers("LOG-1", "LOG-2")) + assert result_properties(collected_item(request, test.__name__)) == ( + ("package", "root"), ("covers", "LOG-1,LOG-2"), - ("source", "tests/e2e/logging/test_x.py:41"), + ("source", f"tests/e2e/test_junit_properties.py:{test.__code__.co_firstlineno}"), ) - def test_attach_is_idempotent(self) -> None: + def test_attach_is_idempotent(self, request: pytest.FixtureRequest) -> None: """Collection can run the hook more than once; a second pass must not double the entries in the report.""" - item = FakeItem("logging/test_x.py::test_bar", ("logging/test_x.py", 40, "test_bar")) + item = collected_item(request, type(self).test_attach_is_idempotent.__name__) attach_result_properties(item) attach_result_properties(item) assert [name for name, _ in item.user_properties] == ["package", "covers", "source"] From 0608f0a00f2c76b50640ed7f4559f4c8551fdc44 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:17:02 +0000 Subject: [PATCH 16/67] fix: reject unknown runtime router settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 22 +++++++++++ litellm/proxy/proxy_server.py | 21 +++++++++- litellm/router.py | 30 ++++---------- litellm/types/router.py | 29 +++++++------- .../proxy/proxy_server/test_routes_config.py | 39 +++++++++++++++++++ .../test_router_retry_policy_update.py | 21 +++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 7 files changed, 125 insertions(+), 39 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 1bd977dd9a9..a7506cb6378 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -38,6 +38,28 @@ DEFAULT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_MAX_TOKENS", 4096)) DEFAULT_ALLOWED_FAILS: Final = int(os.getenv("DEFAULT_ALLOWED_FAILS", 3)) DEFAULT_REDIS_SYNC_INTERVAL: Final = int(os.getenv("DEFAULT_REDIS_SYNC_INTERVAL", 1)) DEFAULT_COOLDOWN_TIME_SECONDS: Final = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECONDS", 5)) +RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( + { + "routing_strategy_args", + "routing_strategy", + "routing_groups", + "allowed_fails", + "cooldown_time", + "num_retries", + "timeout", + "max_retries", + "retry_after", + "fallbacks", + "context_window_fallbacks", + "retry_policy", + "model_group_retry_policy", + "model_group_alias", + "enable_weighted_failover", + "enable_tag_filtering", + "tag_routing_prefix", + "optional_pre_call_checks", + } +) DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5)) DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)) DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 77a80ea0052..9de6b38265a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -39,7 +39,7 @@ from typing import ( import anyio import websockets import websockets.exceptions -from pydantic import BaseModel, Json, JsonValue, ValidationError +from pydantic import BaseModel, Json, JsonValue, TypeAdapter, ValidationError from typing_extensions import NotRequired, ReadOnly, assert_never from litellm._uuid import uuid @@ -60,6 +60,7 @@ from litellm.constants import ( LITELLM_SETTINGS_SAFE_DB_OVERRIDES, LITELLM_UI_ALLOW_HEADERS, LITELLM_UI_SESSION_DURATION, + RUNTIME_UPDATABLE_ROUTER_SETTINGS, ) from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, @@ -16207,6 +16208,7 @@ async def invitation_delete( ) async def update_config( config_info: ConfigYAML, + request: Request, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -16218,6 +16220,23 @@ async def update_config( a side effect of an unrelated update. """ global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client + request_body: Final[Mapping[str, JsonValue]] = TypeAdapter(Mapping[str, JsonValue]).validate_python( + await request.json() + ) + raw_router_settings: Final = request_body.get("router_settings") + if isinstance(raw_router_settings, dict): + unsupported_router_settings: Final = sorted(set(raw_router_settings) - RUNTIME_UPDATABLE_ROUTER_SETTINGS) + if unsupported_router_settings: + raise HTTPException( + status_code=400, + detail={ + "error": ( + f"Unsupported router settings: {', '.join(unsupported_router_settings)} " + "are not runtime-updatable router settings" + ) + }, + ) + try: if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException(status_code=403, detail="Only proxy admins can update config") diff --git a/litellm/router.py b/litellm/router.py index 23d8907fb49..fc4f815170d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -50,6 +50,7 @@ from litellm.constants import ( DEFAULT_HEALTH_CHECK_INTERVAL, DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER, DEFAULT_MAX_LRU_CACHE_SIZE, + RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger @@ -2072,6 +2073,10 @@ class Router: if _callback is None: continue + if self.optional_callbacks is not None and any( + isinstance(callback, type(_callback)) for callback in self.optional_callbacks + ): + continue if self.optional_callbacks is None: self.optional_callbacks = [] self.optional_callbacks.append(_callback) @@ -11331,27 +11336,6 @@ class Router: """ Update the router settings. """ - # only the following settings are allowed to be configured - _allowed_settings: Final = [ - "routing_strategy_args", - "routing_strategy", - "routing_groups", - "allowed_fails", - "cooldown_time", - "num_retries", - "timeout", - "max_retries", - "retry_after", - "fallbacks", - "context_window_fallbacks", - "retry_policy", - "model_group_retry_policy", - "model_group_alias", - "enable_weighted_failover", - "enable_tag_filtering", - "tag_routing_prefix", - ] - _int_settings: Final = [ "timeout", "num_retries", @@ -11364,13 +11348,15 @@ class Router: rebuild_routing_groups = False relink_lar1_from_args = False for var in kwargs: - if var in _allowed_settings: + if var in RUNTIME_UPDATABLE_ROUTER_SETTINGS: if var in _int_settings: _casted_value = int(kwargs[var]) setattr(self, var, _casted_value) elif var == "routing_groups": self._routing_groups_input = kwargs[var] rebuild_routing_groups = True + elif var == "optional_pre_call_checks": + self.add_optional_pre_call_checks(kwargs[var]) elif var == "retry_policy": value = kwargs[var] if isinstance(value, dict): diff --git a/litellm/types/router.py b/litellm/types/router.py index e0957383aac..2a5f264cee3 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -106,6 +106,20 @@ class RetryPolicy(BaseModel): InternalServerErrorRetries: int | None = None +OptionalPreCallChecks = list[ + Literal[ + "prompt_caching", + "router_budget_limiting", + "responses_api_deployment_check", + "deployment_affinity", + "session_affinity", + "forward_client_headers_by_model_group", + "enforce_model_rate_limits", + "encrypted_content_affinity", + ] +] + + class UpdateRouterConfig(BaseModel): """ Set of params that you can modify via `router.update_settings()`. @@ -128,6 +142,7 @@ class UpdateRouterConfig(BaseModel): model_group_alias: dict[str, str | dict] | None = {} enable_tag_filtering: bool | None = None tag_routing_prefix: str | None = None + optional_pre_call_checks: OptionalPreCallChecks | None = None model_config = ConfigDict(protected_namespaces=()) @@ -869,20 +884,6 @@ class FallbackAccessCheck(Protocol): async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: "Router") -> bool: ... -OptionalPreCallChecks = list[ - Literal[ - "prompt_caching", - "router_budget_limiting", - "responses_api_deployment_check", - "deployment_affinity", - "session_affinity", - "forward_client_headers_by_model_group", - "enforce_model_rate_limits", - "encrypted_content_affinity", - ] -] - - class LiteLLM_RouterFileObject(TypedDict, total=False): """ Tracking the litellm params hash, used for mapping the file id to the right model diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index ad3c470acf3..0df8fb663e2 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -60,6 +60,45 @@ def test_config_update_happy_admin(client, auth_as, mock_prisma, monkeypatch): assert normalize(response.json()) == {"message": "Config updated successfully"} +def test_config_update_persists_optional_pre_call_checks(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + fake_proxy_config = MagicMock() + fake_proxy_config.add_deployment = AsyncMock() + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"optional_pre_call_checks": ["prompt_caching"]}}, + ) + + assert response.status_code == 200 + persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"]) + assert persisted["optional_pre_call_checks"] == ["prompt_caching"] + + +def test_config_update_rejects_unknown_router_setting(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"optional_precall_checks": ["prompt_caching"]}}, + ) + + assert response.status_code == 400 + assert "optional_precall_checks" in response.json()["detail"]["error"] + table.upsert.assert_not_called() + + def test_config_update_non_admin_forbidden(client, auth_as, mock_prisma, monkeypatch): """POST /config/update by a non-admin caller is rejected; the error surfaces as a ProxyException with the admin-only message.""" diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 1b98b8c1ae8..1b014cd8401 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -26,8 +26,8 @@ from unittest.mock import AsyncMock, MagicMock import pytest from pydantic import ValidationError - import litellm +from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck from litellm.types.router import RetryPolicy, UpdateRouterConfig # --------------------------------------------------------------------------- @@ -100,6 +100,19 @@ def _build_router() -> litellm.Router: ) +def test_update_settings_adds_optional_pre_call_check_once(): + router = _build_router() + + router.update_settings(num_retries=7, optional_pre_call_checks=["prompt_caching"]) + router.update_settings(optional_pre_call_checks=["prompt_caching"]) + + prompt_caching_callbacks = [ + callback for callback in router.optional_callbacks if isinstance(callback, PromptCachingDeploymentCheck) + ] + assert len(prompt_caching_callbacks) == 1 + assert router.num_retries == 7 + + def test_update_settings_persists_retry_policy_dict(): """When the proxy's ``_add_router_settings_from_db_config`` calls ``llm_router.update_settings(retry_policy={...})`` after reading the @@ -228,7 +241,7 @@ async def test_config_update_persists_and_reads_back_retry_policy(monkeypatch): """The exact global retry_policy save the UI performs must survive the real ``/config/update`` -> DB -> apply -> ``/get/config/callbacks`` path, not snap back to the ``num_retries`` fallback the ticket reported.""" - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import ConfigYAML, LitellmUserRoles, UserAPIKeyAuth router = _build_router() @@ -255,8 +268,12 @@ async def test_config_update_persists_and_reads_back_retry_policy(monkeypatch): RateLimitErrorRetries=7, ) ) + request = MagicMock() + request.json = AsyncMock(return_value={"router_settings": {"retry_policy": posted.model_dump()}}) + await proxy_server.update_config( config_info=ConfigYAML(router_settings=posted), + request=request, user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234"), ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6f044fec3f3..bde7fd611d5 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -37473,6 +37473,8 @@ export interface components { } | null; /** Num Retries */ num_retries?: number | null; + /** Optional Pre Call Checks */ + optional_pre_call_checks?: ("prompt_caching" | "router_budget_limiting" | "responses_api_deployment_check" | "deployment_affinity" | "session_affinity" | "forward_client_headers_by_model_group" | "enforce_model_rate_limits" | "encrypted_content_affinity")[] | null; /** Retry After */ retry_after?: number | null; retry_policy?: components["schemas"]["RetryPolicy"] | null; From cb511f70ccbee8cc9257b52cc6ad5d7721219ce3 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:20:51 +0000 Subject: [PATCH 17/67] fix: preserve config update authorization order Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 44 +++++++++---------- litellm/proxy/proxy_server.py | 34 +++++++------- .../proxy/proxy_server/test_routes_config.py | 19 +++++++- .../test_router_retry_policy_update.py | 3 +- 4 files changed, 59 insertions(+), 41 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index a7506cb6378..5b44e8b5f51 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -9,6 +9,28 @@ DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT" AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5)) ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000 +RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( + { + "routing_strategy_args", + "routing_strategy", + "routing_groups", + "allowed_fails", + "cooldown_time", + "num_retries", + "timeout", + "max_retries", + "retry_after", + "fallbacks", + "context_window_fallbacks", + "retry_policy", + "model_group_retry_policy", + "model_group_alias", + "enable_weighted_failover", + "enable_tag_filtering", + "tag_routing_prefix", + "optional_pre_call_checks", + } +) DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) @@ -38,28 +60,6 @@ DEFAULT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_MAX_TOKENS", 4096)) DEFAULT_ALLOWED_FAILS: Final = int(os.getenv("DEFAULT_ALLOWED_FAILS", 3)) DEFAULT_REDIS_SYNC_INTERVAL: Final = int(os.getenv("DEFAULT_REDIS_SYNC_INTERVAL", 1)) DEFAULT_COOLDOWN_TIME_SECONDS: Final = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECONDS", 5)) -RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( - { - "routing_strategy_args", - "routing_strategy", - "routing_groups", - "allowed_fails", - "cooldown_time", - "num_retries", - "timeout", - "max_retries", - "retry_after", - "fallbacks", - "context_window_fallbacks", - "retry_policy", - "model_group_retry_policy", - "model_group_alias", - "enable_weighted_failover", - "enable_tag_filtering", - "tag_routing_prefix", - "optional_pre_call_checks", - } -) DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5)) DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)) DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9de6b38265a..dbeb8486539 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16220,27 +16220,27 @@ async def update_config( a side effect of an unrelated update. """ global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client - request_body: Final[Mapping[str, JsonValue]] = TypeAdapter(Mapping[str, JsonValue]).validate_python( - await request.json() - ) - raw_router_settings: Final = request_body.get("router_settings") - if isinstance(raw_router_settings, dict): - unsupported_router_settings: Final = sorted(set(raw_router_settings) - RUNTIME_UPDATABLE_ROUTER_SETTINGS) - if unsupported_router_settings: - raise HTTPException( - status_code=400, - detail={ - "error": ( - f"Unsupported router settings: {', '.join(unsupported_router_settings)} " - "are not runtime-updatable router settings" - ) - }, - ) - try: if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException(status_code=403, detail="Only proxy admins can update config") + request_body: Final[Mapping[str, JsonValue]] = TypeAdapter(Mapping[str, JsonValue]).validate_python( + await request.json() + ) + raw_router_settings: Final = request_body.get("router_settings") + if isinstance(raw_router_settings, dict): + unsupported_router_settings: Final = sorted(set(raw_router_settings) - RUNTIME_UPDATABLE_ROUTER_SETTINGS) + if unsupported_router_settings: + raise HTTPException( + status_code=400, + detail={ + "error": ( + f"Unsupported router settings: {', '.join(unsupported_router_settings)} " + "are not runtime-updatable router settings" + ) + }, + ) + if prisma_client is None: raise Exception("No DB Connected") diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index 0df8fb663e2..4b0954c350a 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -95,10 +95,27 @@ def test_config_update_rejects_unknown_router_setting(client, auth_as, mock_pris ) assert response.status_code == 400 - assert "optional_precall_checks" in response.json()["detail"]["error"] + assert "optional_precall_checks" in response.json()["error"]["message"] table.upsert.assert_not_called() +def test_config_update_unknown_router_setting_non_admin_forbidden(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.INTERNAL_USER): + response = client.post( + "/config/update", + json={"router_settings": {"optional_precall_checks": ["prompt_caching"]}}, + ) + + assert response.status_code == 403 + assert "admin" in response.json()["error"]["message"].lower() + + def test_config_update_non_admin_forbidden(client, auth_as, mock_prisma, monkeypatch): """POST /config/update by a non-admin caller is rejected; the error surfaces as a ProxyException with the admin-only message.""" diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 1b014cd8401..e386eebf3d9 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -26,6 +26,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from pydantic import ValidationError + import litellm from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck from litellm.types.router import RetryPolicy, UpdateRouterConfig @@ -241,7 +242,7 @@ async def test_config_update_persists_and_reads_back_retry_policy(monkeypatch): """The exact global retry_policy save the UI performs must survive the real ``/config/update`` -> DB -> apply -> ``/get/config/callbacks`` path, not snap back to the ``num_retries`` fallback the ticket reported.""" - from litellm.proxy import proxy_server + import litellm.proxy.proxy_server as proxy_server from litellm.proxy._types import ConfigYAML, LitellmUserRoles, UserAPIKeyAuth router = _build_router() From 29f0110fe08d5b8f4798e20eed6d2368e4cada2d Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:29:41 +0000 Subject: [PATCH 18/67] test: pass request to config update test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/proxy_unit_tests/test_proxy_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 47554913419..54cce9cdd78 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -3076,7 +3076,9 @@ async def test_update_config_success_callback_normalization(): admin_user = UserAPIKeyAuth( user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test" ) - await proxy_server.update_config(config_update, user_api_key_dict=admin_user) + request = MagicMock() + request.json = AsyncMock(return_value={"litellm_settings": {"success_callback": ["SQS", "sQs"]}}) + await proxy_server.update_config(config_update, request=request, user_api_key_dict=admin_user) assert ( "litellm_settings" in upserted From e67f98feb1cb1758e253beaf005eaaad44ab3abe Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:44:17 +0000 Subject: [PATCH 19/67] fix: reconcile runtime pre-call checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 30 +++++++++++++- .../test_router_retry_policy_update.py | 40 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index fc4f815170d..e2865542e89 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -355,6 +355,13 @@ _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" +_RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS: Final[Mapping[str, type[CustomLogger]]] = MappingProxyType( + { + "prompt_caching": PromptCachingDeploymentCheck, + "enforce_model_rate_limits": ModelRateLimitingCheck, + } +) + def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool: for chunk in chunks: @@ -2082,6 +2089,27 @@ class Router: self.optional_callbacks.append(_callback) litellm.logging_callback_manager.add_litellm_callback(_callback) + def set_optional_pre_call_checks(self, optional_pre_call_checks: OptionalPreCallChecks | None) -> None: + if optional_pre_call_checks is None: + return + requested: Final = frozenset(optional_pre_call_checks) + for name, callback_cls in _RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS.items(): + if name not in requested: + self._remove_optional_callbacks_of_type(callback_cls) + self.add_optional_pre_call_checks(optional_pre_call_checks) + + def _remove_optional_callbacks_of_type(self, callback_cls: type[CustomLogger]) -> None: + if self.optional_callbacks is None: + return + removed: Final = [cb for cb in self.optional_callbacks if isinstance(cb, callback_cls)] + if not removed: + return + self.optional_callbacks = [cb for cb in self.optional_callbacks if not isinstance(cb, callback_cls)] + for cb in removed: + litellm.logging_callback_manager.remove_callback_from_list_by_object( + litellm.callbacks, cb, require_self=False + ) + def print_deployment(self, deployment: dict): """ returns a copy of the deployment with the api key masked @@ -11356,7 +11384,7 @@ class Router: self._routing_groups_input = kwargs[var] rebuild_routing_groups = True elif var == "optional_pre_call_checks": - self.add_optional_pre_call_checks(kwargs[var]) + self.set_optional_pre_call_checks(kwargs[var]) elif var == "retry_policy": value = kwargs[var] if isinstance(value, dict): diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index e386eebf3d9..2c23d0da7e7 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -28,6 +28,8 @@ from pydantic import ValidationError import litellm +from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck from litellm.types.router import RetryPolicy, UpdateRouterConfig @@ -114,6 +116,44 @@ def test_update_settings_adds_optional_pre_call_check_once(): assert router.num_retries == 7 +def test_update_settings_clears_omitted_toggleable_pre_call_checks(): + router = _build_router() + + router.update_settings(optional_pre_call_checks=["prompt_caching"]) + router.update_settings(optional_pre_call_checks=[]) + + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + + +def test_update_settings_replaces_toggleable_pre_call_checks(): + router = _build_router() + + router.update_settings(optional_pre_call_checks=["prompt_caching"]) + router.update_settings(optional_pre_call_checks=["enforce_model_rate_limits"]) + + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + assert any(isinstance(callback, ModelRateLimitingCheck) for callback in (router.optional_callbacks or [])) + + +@pytest.mark.asyncio +async def test_update_settings_preserves_router_budget_limiting_when_omitted(monkeypatch): + async def _disable_periodic_sync(*args, **kwargs): + return None + + monkeypatch.setattr( + "litellm.router_strategy.budget_limiter.RouterBudgetLimiting.periodic_sync_in_memory_spend_with_redis", + _disable_periodic_sync, + ) + router = _build_router() + + router.add_optional_pre_call_checks(["router_budget_limiting"]) + router.update_settings(optional_pre_call_checks=[]) + + assert any(isinstance(callback, RouterBudgetLimiting) for callback in (router.optional_callbacks or [])) + + def test_update_settings_persists_retry_policy_dict(): """When the proxy's ``_add_router_settings_from_db_config`` calls ``llm_router.update_settings(retry_policy={...})`` after reading the From 7b86b7f4cd7576aeba56d9721f28002a4e5c6383 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:45:45 +0000 Subject: [PATCH 20/67] test: isolate router callback state Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_router_retry_policy_update.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 2c23d0da7e7..db710f76887 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -21,6 +21,7 @@ This file pins both halves of the fix. import json from dataclasses import dataclass +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -33,6 +34,14 @@ from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRat from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck from litellm.types.router import RetryPolicy, UpdateRouterConfig + +@pytest.fixture(autouse=True) +def isolate_litellm_callbacks(): + callbacks_before: Final = litellm.callbacks.copy() + yield + litellm.callbacks = callbacks_before + + # --------------------------------------------------------------------------- # UpdateRouterConfig schema membership (LIT-3152 part 1) # --------------------------------------------------------------------------- From 3dea586d3be34c52156fe764051dd8d750d4570b Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:48:09 +0000 Subject: [PATCH 21/67] test: cover runtime callback reconciliation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_router_retry_policy_update.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index db710f76887..9e0bb0b9bef 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -135,6 +135,26 @@ def test_update_settings_clears_omitted_toggleable_pre_call_checks(): assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) +def test_set_optional_pre_call_checks_reconciles_callback_types(): + router = _build_router() + + router.set_optional_pre_call_checks(["prompt_caching"]) + router.set_optional_pre_call_checks([]) + + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + + +def test_remove_optional_pre_call_check_removes_local_and_global_callbacks(): + router = _build_router() + + router.set_optional_pre_call_checks(["prompt_caching"]) + router._remove_optional_callbacks_of_type(PromptCachingDeploymentCheck) + + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + + def test_update_settings_replaces_toggleable_pre_call_checks(): router = _build_router() From 70a4f74a0d65bd89a9e9127076efd4301b987a29 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 02:56:23 +0000 Subject: [PATCH 22/67] test: allow callback state fixture mutation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_router_retry_policy_update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 9e0bb0b9bef..26251290176 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -39,7 +39,7 @@ from litellm.types.router import RetryPolicy, UpdateRouterConfig def isolate_litellm_callbacks(): callbacks_before: Final = litellm.callbacks.copy() yield - litellm.callbacks = callbacks_before + litellm.callbacks = callbacks_before # test-quality-ok: required callback-state restoration fixture # --------------------------------------------------------------------------- From 1d3e26fd98b40d40f498e14b2470e8acc79fb9f6 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 03:28:04 +0000 Subject: [PATCH 23/67] fix: preserve shared optional callbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 19 +++++---- .../test_router_retry_policy_update.py | 41 ++++++++++++++++++- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index e2865542e89..e7588d8ad5a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2099,16 +2099,19 @@ class Router: self.add_optional_pre_call_checks(optional_pre_call_checks) def _remove_optional_callbacks_of_type(self, callback_cls: type[CustomLogger]) -> None: - if self.optional_callbacks is None: + if self.optional_callbacks is None or not any(type(cb) is callback_cls for cb in self.optional_callbacks): return - removed: Final = [cb for cb in self.optional_callbacks if isinstance(cb, callback_cls)] - if not removed: + self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls] + if any( + router is not self and any(type(cb) is callback_cls for cb in (router.optional_callbacks or [])) + for router in tuple(_live_routers) + ): return - self.optional_callbacks = [cb for cb in self.optional_callbacks if not isinstance(cb, callback_cls)] - for cb in removed: - litellm.logging_callback_manager.remove_callback_from_list_by_object( - litellm.callbacks, cb, require_self=False - ) + for cb in tuple(litellm.callbacks): + if type(cb) is callback_cls: + litellm.logging_callback_manager.remove_callback_from_list_by_object( + litellm.callbacks, cb, require_self=False + ) def print_deployment(self, deployment: dict): """ diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 26251290176..be568134763 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -151,8 +151,45 @@ def test_remove_optional_pre_call_check_removes_local_and_global_callbacks(): router.set_optional_pre_call_checks(["prompt_caching"]) router._remove_optional_callbacks_of_type(PromptCachingDeploymentCheck) - assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) - assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router.optional_callbacks or [])) + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + +def test_remove_optional_pre_call_check_keeps_global_callback_for_another_router(): + router_a = _build_router() + router_b = _build_router() + + router_a.update_settings(optional_pre_call_checks=["prompt_caching"]) + router_b.update_settings(optional_pre_call_checks=["prompt_caching"]) + + router_a.update_settings(optional_pre_call_checks=[]) + + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router_a.optional_callbacks or [])) + assert any(type(callback) is PromptCachingDeploymentCheck for callback in (router_b.optional_callbacks or [])) + assert any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + router_b.update_settings(optional_pre_call_checks=[]) + + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router_b.optional_callbacks or [])) + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + +def test_remove_optional_pre_call_check_keeps_global_callback_when_second_router_clears_first(): + router_a = _build_router() + router_b = _build_router() + + router_a.update_settings(optional_pre_call_checks=["prompt_caching"]) + router_b.update_settings(optional_pre_call_checks=["prompt_caching"]) + + router_b.update_settings(optional_pre_call_checks=[]) + + assert any(type(callback) is PromptCachingDeploymentCheck for callback in (router_a.optional_callbacks or [])) + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router_b.optional_callbacks or [])) + assert any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + router_a.update_settings(optional_pre_call_checks=[]) + + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) def test_update_settings_replaces_toggleable_pre_call_checks(): From 974b331a4d09e2883d6fe84bb87ce57cc49ab5bb Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 06:15:52 +0000 Subject: [PATCH 24/67] fix: accept persistable router settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/proxy/proxy_server.py | 32 +++++++++----- .../proxy/proxy_server/test_routes_config.py | 43 +++++++++++++++++++ 3 files changed, 65 insertions(+), 11 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 5b44e8b5f51..b6cb6b32187 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -31,6 +31,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( "optional_pre_call_checks", } ) +ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset({"model_list", "search_tools"}) DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dbeb8486539..ae5e08ab566 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -254,6 +254,7 @@ from litellm.constants import ( PROXY_BUDGET_RESCHEDULER_MAX_TIME, PROXY_BUDGET_RESCHEDULER_MIN_TIME, PROXY_CONFIG_RELOAD_INTERVAL_SECONDS, + ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG, USER_SPEND_ALERTS_JOB_ID, WEEKLY_SPEND_REPORT_JOB_ID, ) @@ -5711,13 +5712,9 @@ class ProxyConfig: router_settings: Final = config.get("router_settings", None) if router_settings and isinstance(router_settings, dict): - # model list and search_tools already set - exclude_args: Final = { - "model_list", - "search_tools", - } - - available_args: Final = [x for x in litellm.Router.get_valid_args() if x not in exclude_args] + available_args: Final = [ + x for x in litellm.Router.get_valid_args() if x not in ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG + ] for k, v in router_settings.items(): if k in available_args: @@ -16229,14 +16226,17 @@ async def update_config( ) raw_router_settings: Final = request_body.get("router_settings") if isinstance(raw_router_settings, dict): - unsupported_router_settings: Final = sorted(set(raw_router_settings) - RUNTIME_UPDATABLE_ROUTER_SETTINGS) + supported_router_settings: Final = RUNTIME_UPDATABLE_ROUTER_SETTINGS | ( + frozenset(litellm.Router.get_valid_args()) - ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG + ) + unsupported_router_settings: Final = sorted(set(raw_router_settings) - supported_router_settings) if unsupported_router_settings: raise HTTPException( status_code=400, detail={ "error": ( f"Unsupported router settings: {', '.join(unsupported_router_settings)} " - "are not runtime-updatable router settings" + "are not valid router settings" ) }, ) @@ -16342,10 +16342,20 @@ async def update_config( ) # router_settings: merge existing + request, request wins. - if config_info.router_settings is not None: + if isinstance(raw_router_settings, dict): existing = await _read_section("router_settings") before_router_settings: Final = copy.deepcopy(existing) - updates = config_info.router_settings.dict(exclude_none=True) + typed_router_settings: Final = ( + config_info.router_settings.dict(exclude_none=True) + if config_info.router_settings is not None + else {} + ) + raw_router_settings_without_none: Final = { + key: value + for key, value in raw_router_settings.items() + if key not in typed_router_settings and value is not None + } + updates: Final = {**typed_router_settings, **raw_router_settings_without_none} new_router_settings: Final = {**existing, **updates} await _upsert_section("router_settings", new_router_settings) asyncio.create_task( diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index 4b0954c350a..6166513d229 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -81,6 +81,49 @@ def test_config_update_persists_optional_pre_call_checks(client, auth_as, mock_p assert persisted["optional_pre_call_checks"] == ["prompt_caching"] +def test_config_update_persists_model_group_affinity_config(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + fake_proxy_config = MagicMock() + fake_proxy_config.add_deployment = AsyncMock() + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + model_group_affinity_config = {"gpt-4": ["session_affinity"]} + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"model_group_affinity_config": model_group_affinity_config}}, + ) + + assert response.status_code == 200 + persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"]) + assert persisted["model_group_affinity_config"] == model_group_affinity_config + + +def test_config_update_persists_disable_cooldowns(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + fake_proxy_config = MagicMock() + fake_proxy_config.add_deployment = AsyncMock() + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"disable_cooldowns": True}}, + ) + + assert response.status_code == 200 + persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"]) + assert persisted["disable_cooldowns"] is True + + def test_config_update_rejects_unknown_router_setting(client, auth_as, mock_prisma, monkeypatch): from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles From 95e09db661471d8fbdaa1a93f79bf8e49ecdd0eb Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 06:56:27 +0000 Subject: [PATCH 25/67] style: apply ruff format to router settings merge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ae5e08ab566..1bdf2ba9987 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16346,9 +16346,7 @@ async def update_config( existing = await _read_section("router_settings") before_router_settings: Final = copy.deepcopy(existing) typed_router_settings: Final = ( - config_info.router_settings.dict(exclude_none=True) - if config_info.router_settings is not None - else {} + config_info.router_settings.dict(exclude_none=True) if config_info.router_settings is not None else {} ) raw_router_settings_without_none: Final = { key: value From aba9644297e4e709232775031cb03f028dd1cbd6 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 07:10:04 +0000 Subject: [PATCH 26/67] fix: avoid router settings update name collision Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1bdf2ba9987..6632b209905 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16353,8 +16353,8 @@ async def update_config( for key, value in raw_router_settings.items() if key not in typed_router_settings and value is not None } - updates: Final = {**typed_router_settings, **raw_router_settings_without_none} - new_router_settings: Final = {**existing, **updates} + router_settings_updates: Final = {**typed_router_settings, **raw_router_settings_without_none} + new_router_settings: Final = {**existing, **router_settings_updates} await _upsert_section("router_settings", new_router_settings) asyncio.create_task( create_config_audit_log( From 385957e830c6b5edae9455dae8361527908cd4bb Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 07:49:25 +0000 Subject: [PATCH 27/67] fix: reject constructor-managed router settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 11 +++++- .../proxy/proxy_server/test_routes_config.py | 36 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/litellm/constants.py b/litellm/constants.py index b6cb6b32187..1c1939bd350 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -31,7 +31,16 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( "optional_pre_call_checks", } ) -ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset({"model_list", "search_tools"}) +ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset( + { + "model_list", + "search_tools", + "assistants_config", + "router_general_settings", + "ignore_invalid_deployments", + "fallback_access_check", + } +) DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index 6166513d229..dcb63b8ca82 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -124,6 +124,42 @@ def test_config_update_persists_disable_cooldowns(client, auth_as, mock_prisma, assert persisted["disable_cooldowns"] is True +def test_config_update_rejects_assistants_config(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"assistants_config": {"enabled": True}}}, + ) + + assert response.status_code == 400 + assert "assistants_config" in response.json()["error"]["message"] + table.upsert.assert_not_called() + + +def test_config_update_rejects_router_general_settings(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"router_general_settings": {"async_only_mode": True}}}, + ) + + assert response.status_code == 400 + assert "router_general_settings" in response.json()["error"]["message"] + table.upsert.assert_not_called() + + def test_config_update_rejects_unknown_router_setting(client, auth_as, mock_prisma, monkeypatch): from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles From a7836ede15bb4f62d8f44bdb991402a9829727e3 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 14:48:51 +0000 Subject: [PATCH 28/67] fix(models): absorb open registry PRs: govcloud bedrock and mantle, azure gov, openai tiered long-context, scaleway, together qwen3.8, azure ai cache and kimi k2.7 code, azure mai deprecations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 605 +++++++++++++++++- model_prices_and_context_window.json | 605 +++++++++++++++++- .../llm_cost_calc/test_llm_cost_calc_utils.py | 6 +- ...penai_service_tier_long_context_pricing.py | 156 +++++ whitelisted_bedrock_models.txt | 14 + 5 files changed, 1341 insertions(+), 45 deletions(-) create mode 100644 tests/test_litellm/test_openai_service_tier_long_context_pricing.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c710db1a749..87d348f4752 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9643,7 +9643,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2.5-Flash": { "input_cost_per_image_token": 1.75e-06, @@ -9656,7 +9657,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2e": { "deprecation_date": "2026-08-15", @@ -10155,7 +10157,9 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "cache_read_input_token_cost": 1.45e-07, + "supports_prompt_caching": true }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", @@ -10169,18 +10173,20 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "cache_read_input_token_cost": 2.8e-08, + "supports_prompt_caching": true }, "azure_ai/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, + "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.1e-07, + "output_cost_per_token": 1.32e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_prompt_caching": true, @@ -10400,11 +10406,13 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supports_function_calling": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true }, "azure_ai/kimi-k2.6": { "deprecation_date": "2027-04-16", @@ -10415,7 +10423,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k2-6-in-microsoft-foundry/4513125", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supported_modalities": [ "text", "image" @@ -10426,7 +10434,9 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1.6e-07, + "supports_prompt_caching": true }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, @@ -12110,7 +12120,7 @@ "max_output_tokens": 2048, "max_tokens": 2048, "mode": "chat", - "output_cost_per_token": 2.65e-06, + "output_cost_per_token": 6e-07, "supports_pdf_input": true }, "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": { @@ -29098,16 +29108,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29119,6 +29132,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29161,16 +29175,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29182,6 +29199,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29225,16 +29243,19 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, "cache_creation_input_token_cost_flex": 1.25e-06, "cache_creation_input_token_cost_priority": 5e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, @@ -29246,6 +29267,7 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, "output_cost_per_token_above_272k_tokens_flex": 9e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, @@ -29288,16 +29310,19 @@ "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, "cache_creation_input_token_cost_flex": 1.25e-07, "cache_creation_input_token_cost_priority": 5e-07, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_above_272k_tokens_flex": 2e-07, + "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, @@ -29309,6 +29334,7 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, "output_cost_per_token_above_272k_tokens_flex": 9e-07, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, @@ -29548,7 +29574,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, @@ -29602,7 +29631,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -29751,7 +29783,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, @@ -29800,7 +29835,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, @@ -29849,7 +29887,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, @@ -29898,7 +29938,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, @@ -41350,13 +41392,13 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/Qwen/Qwen3.8-2.4T-A95B": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, "max_tokens": 1010000, "mode": "chat", - "output_cost_per_token": 6.25e-06, + "output_cost_per_token": 6e-06, "source": "https://docs.together.ai/docs/serverless-models", "supports_prompt_caching": true }, @@ -57558,5 +57600,526 @@ "supported_endpoints": [ "/v1/audio/transcriptions" ] + }, + "scaleway/glm-5.2": { + "input_cost_per_token": 1.8e-06, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 5.5e-06, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": false + }, + "scaleway/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": false + }, + "azure_ai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "deprecation_date": "2026-10-03", + "input_cost_per_token": 9.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-west-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_above_272k_tokens": 5.28e-06, + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-06, + "cache_read_input_token_cost": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-07, + "output_cost_per_token": 1.584e-05, + "output_cost_per_token_above_272k_tokens": 2.376e-05 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-07, + "input_cost_per_token_above_272k_tokens": 5.28e-07, + "cache_creation_input_token_cost": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-07, + "cache_read_input_token_cost": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-08, + "output_cost_per_token": 1.584e-06, + "output_cost_per_token_above_272k_tokens": 2.376e-06 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.3": { + "use_openai_responses_path": true, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/", + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.4e-07 + }, + "bedrock_mantle/us-gov-east-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "azure/us-gov/gpt-5.1": { + "cache_read_input_token_cost": 1.71875e-07, + "default_reasoning_effort": "none", + "deprecation_date": "2027-05-15", + "input_cost_per_token": 1.71875e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.375e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us-gov/o3-mini": { + "cache_read_input_token_cost": 7.57e-07, + "deprecation_date": "2026-10-01", + "input_cost_per_token": 1.513e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 6.05e-06, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "azure/us-gov/text-embedding-3-large": { + "deprecation_date": "2028-02-09", + "input_cost_per_token": 1.63e-07, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "azure/us-gov/text-embedding-3-small": { + "deprecation_date": "2028-02-09", + "input_cost_per_token": 2.5e-08, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c710db1a749..87d348f4752 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9643,7 +9643,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2.5-Flash": { "input_cost_per_image_token": 1.75e-06, @@ -9656,7 +9657,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2e": { "deprecation_date": "2026-08-15", @@ -10155,7 +10157,9 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "cache_read_input_token_cost": 1.45e-07, + "supports_prompt_caching": true }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", @@ -10169,18 +10173,20 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "cache_read_input_token_cost": 2.8e-08, + "supports_prompt_caching": true }, "azure_ai/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, + "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.1e-07, + "output_cost_per_token": 1.32e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_prompt_caching": true, @@ -10400,11 +10406,13 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supports_function_calling": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true }, "azure_ai/kimi-k2.6": { "deprecation_date": "2027-04-16", @@ -10415,7 +10423,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k2-6-in-microsoft-foundry/4513125", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supported_modalities": [ "text", "image" @@ -10426,7 +10434,9 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1.6e-07, + "supports_prompt_caching": true }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, @@ -12110,7 +12120,7 @@ "max_output_tokens": 2048, "max_tokens": 2048, "mode": "chat", - "output_cost_per_token": 2.65e-06, + "output_cost_per_token": 6e-07, "supports_pdf_input": true }, "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": { @@ -29098,16 +29108,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29119,6 +29132,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29161,16 +29175,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29182,6 +29199,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29225,16 +29243,19 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, "cache_creation_input_token_cost_flex": 1.25e-06, "cache_creation_input_token_cost_priority": 5e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, @@ -29246,6 +29267,7 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, "output_cost_per_token_above_272k_tokens_flex": 9e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, @@ -29288,16 +29310,19 @@ "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, "cache_creation_input_token_cost_flex": 1.25e-07, "cache_creation_input_token_cost_priority": 5e-07, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_above_272k_tokens_flex": 2e-07, + "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, @@ -29309,6 +29334,7 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, "output_cost_per_token_above_272k_tokens_flex": 9e-07, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, @@ -29548,7 +29574,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, @@ -29602,7 +29631,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -29751,7 +29783,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, @@ -29800,7 +29835,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, @@ -29849,7 +29887,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, @@ -29898,7 +29938,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, @@ -41350,13 +41392,13 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/Qwen/Qwen3.8-2.4T-A95B": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, "max_tokens": 1010000, "mode": "chat", - "output_cost_per_token": 6.25e-06, + "output_cost_per_token": 6e-06, "source": "https://docs.together.ai/docs/serverless-models", "supports_prompt_caching": true }, @@ -57558,5 +57600,526 @@ "supported_endpoints": [ "/v1/audio/transcriptions" ] + }, + "scaleway/glm-5.2": { + "input_cost_per_token": 1.8e-06, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 5.5e-06, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": false + }, + "scaleway/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": false + }, + "azure_ai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "deprecation_date": "2026-10-03", + "input_cost_per_token": 9.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-west-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_above_272k_tokens": 5.28e-06, + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-06, + "cache_read_input_token_cost": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-07, + "output_cost_per_token": 1.584e-05, + "output_cost_per_token_above_272k_tokens": 2.376e-05 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-07, + "input_cost_per_token_above_272k_tokens": 5.28e-07, + "cache_creation_input_token_cost": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-07, + "cache_read_input_token_cost": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-08, + "output_cost_per_token": 1.584e-06, + "output_cost_per_token_above_272k_tokens": 2.376e-06 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.3": { + "use_openai_responses_path": true, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/", + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.4e-07 + }, + "bedrock_mantle/us-gov-east-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "azure/us-gov/gpt-5.1": { + "cache_read_input_token_cost": 1.71875e-07, + "default_reasoning_effort": "none", + "deprecation_date": "2027-05-15", + "input_cost_per_token": 1.71875e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.375e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us-gov/o3-mini": { + "cache_read_input_token_cost": 7.57e-07, + "deprecation_date": "2026-10-01", + "input_cost_per_token": 1.513e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 6.05e-06, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "azure/us-gov/text-embedding-3-large": { + "deprecation_date": "2028-02-09", + "input_cost_per_token": 1.63e-07, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "azure/us-gov/text-embedding-3-small": { + "deprecation_date": "2028-02-09", + "input_cost_per_token": 2.5e-08, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 } } diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 0e1c832ebf5..5c8de19a7e9 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1522,7 +1522,7 @@ def test_gpt_5_6_alias_prices_match_sol(local_model_cost_map): sol = litellm.model_cost["gpt-5.6-sol"] cost_fields = sorted(field for field in sol if "cost" in field) - assert len(cost_fields) == 23 + assert len(cost_fields) == 27 for field in cost_fields: assert alias.get(field) == sol.get(field), field @@ -4039,8 +4039,8 @@ def test_fast_service_tier_matches_priority_above_the_context_threshold(_local_m ) assert fast == priority - assert fast[0] == pytest.approx(300_000 * 8e-06, rel=1e-9) - assert fast[1] == pytest.approx(1_000 * 3e-05, rel=1e-9) + assert fast[0] == pytest.approx(300_000 * 1.6e-05, rel=1e-9) + assert fast[1] == pytest.approx(1_000 * 6e-05, rel=1e-9) def test_priority_reasoning_tokens_bill_at_the_priority_output_rate(_local_model_cost_map): diff --git a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py new file mode 100644 index 00000000000..c0860a5b55f --- /dev/null +++ b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py @@ -0,0 +1,156 @@ +import json +from functools import lru_cache +from pathlib import Path + +import pytest + +import litellm + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +FLEX_LONG_CONTEXT = { + "gpt-5.4": { + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07, + }, + "gpt-5.4-pro": { + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135, + }, + "gpt-5.5": { + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07, + }, +} + +PRIORITY_LONG_CONTEXT = { + "gpt-5.6": { + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, + }, + "gpt-5.6-sol": { + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, + }, + "gpt-5.6-terra": { + "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + }, + "gpt-5.6-luna": { + "input_cost_per_token_above_272k_tokens_priority": 8e-07, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, + }, +} + +EXPECTED = {**FLEX_LONG_CONTEXT, **PRIORITY_LONG_CONTEXT} + +NO_PUBLISHED_PRIORITY_LONG_CONTEXT = ("gpt-5.4", "gpt-5.5") + + +@pytest.fixture(autouse=True) +def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + +@lru_cache(maxsize=2) +def _load(path: Path) -> dict[str, dict[str, object]]: + with open(path) as f: + return json.load(f) + + +@pytest.mark.parametrize("path", [MAIN_PATH, BACKUP_PATH], ids=["main", "backup"]) +@pytest.mark.parametrize("model", sorted(EXPECTED)) +def test_service_tier_long_context_rates_are_published(model: str, path: Path) -> None: + """Each tier must carry its own above-272K rates, in both price files.""" + info = _load(path).get(model) + assert info is not None, f"{model} not found in {path.name}" + for key, expected in EXPECTED[model].items(): + assert info.get(key) == pytest.approx(expected), f"{model}.{key} is {info.get(key)!r}, expected {expected!r}" + + +@pytest.mark.parametrize("model", sorted(EXPECTED)) +def test_tier_long_context_rate_is_half_or_double_the_standard(model: str) -> None: + """Flex is half the standard long-context rate; priority is double it.""" + info = _load(MAIN_PATH)[model] + tier = "flex" if model in FLEX_LONG_CONTEXT else "priority" + ratio = 0.5 if tier == "flex" else 2.0 + for base in ("input_cost_per_token", "output_cost_per_token"): + standard = info[f"{base}_above_272k_tokens"] + tiered = info[f"{base}_above_272k_tokens_{tier}"] + assert tiered == pytest.approx(standard * ratio), ( + f"{model}.{base}_above_272k_tokens_{tier} is {tiered!r}, " + f"expected {ratio}x the standard long-context rate {standard!r}" + ) + + +@pytest.mark.parametrize("model", NO_PUBLISHED_PRIORITY_LONG_CONTEXT) +def test_no_priority_long_context_rates_where_openai_publishes_none(model: str) -> None: + """Guard against back-filling a rate OpenAI does not publish.""" + info = _load(MAIN_PATH)[model] + assert "input_cost_per_token_above_272k_tokens_priority" not in info + + +LONG_CONTEXT_PROMPT_TOKENS = 300_000 +COMPLETION_TOKENS = 1_000 + +TIERED_COST_CASES = [ + ("gpt-5.4", "flex", 2.5e-06, 1.125e-05), + ("gpt-5.4-pro", "flex", 3e-05, 0.000135), + ("gpt-5.5", "flex", 5e-06, 2.25e-05), + ("gpt-5.6", "priority", 1.6e-05, 6e-05), + ("gpt-5.6-sol", "priority", 1.6e-05, 6e-05), + ("gpt-5.6-terra", "priority", 8e-06, 3.6e-05), + ("gpt-5.6-luna", "priority", 8e-07, 3.6e-06), +] + + +@pytest.mark.parametrize("model,tier,input_rate,output_rate", TIERED_COST_CASES) +def test_cost_per_token_bills_long_context_at_the_tier_rate( + model: str, tier: str, input_rate: float, output_rate: float +) -> None: + """A prompt over 272K on flex or priority must bill at that tier's long-context rate.""" + input_cost, output_cost = litellm.cost_per_token( + model=model, + prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + service_tier=tier, + ) + assert input_cost == pytest.approx(LONG_CONTEXT_PROMPT_TOKENS * input_rate) + assert output_cost == pytest.approx(COMPLETION_TOKENS * output_rate) + + +@pytest.mark.parametrize("model,tier,input_rate,output_rate", TIERED_COST_CASES) +def test_cost_per_token_tier_differs_from_the_standard_long_context_cost( + model: str, tier: str, input_rate: float, output_rate: float +) -> None: + """Flex halves the standard long-context bill and priority doubles it.""" + ratio = 0.5 if tier == "flex" else 2.0 + standard = sum( + litellm.cost_per_token( + model=model, + prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + ) + ) + tiered = sum( + litellm.cost_per_token( + model=model, + prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + service_tier=tier, + ) + ) + assert tiered == pytest.approx(standard * ratio) diff --git a/whitelisted_bedrock_models.txt b/whitelisted_bedrock_models.txt index 7e20081988d..8753d7c3c77 100644 --- a/whitelisted_bedrock_models.txt +++ b/whitelisted_bedrock_models.txt @@ -217,3 +217,17 @@ bedrock/us-east-1/zai.glm-5 bedrock/us-west-2/zai.glm-5 bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0 bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0 +bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b +bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2 +bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b +bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0 +bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0 +bedrock/us-gov-west-1/anthropic.claude-sonnet-5 +bedrock/us-gov-west-1/anthropic.claude-opus-4-8 +bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b +bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2 +bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b +bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0 +bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0 +bedrock/us-gov-east-1/anthropic.claude-sonnet-5 +bedrock/us-gov-east-1/anthropic.claude-opus-4-8 From 6b83b16559e5ceb4904121bcb90623a5f9f7115c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 08:04:14 -0700 Subject: [PATCH 29/67] feat(gemini): day-0 pricing for gemini-3.8-flash Gemini 3.8 Flash launches today with the same promotional pricing, limits, and thinking settings as Gemini 3.7 Flash, so the gemini/, vertex_ai/, and bare cost map entries mirror the 3.7 Flash ones. Regression tests lock the launch prices, the 4096-token cache minimum, and the gemini-3 thought signature gate in for the new model. --- ...odel_prices_and_context_window_backup.json | 173 ++++++++++++++++++ model_prices_and_context_window.json | 173 ++++++++++++++++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 45 +++++ .../test_vertex_ai_gemini_transformation.py | 3 + tests/test_litellm/test_utils.py | 1 + 5 files changed, 395 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a3cfb300ea6..cc828e126ad 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23514,6 +23514,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "vertex_ai/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -25351,6 +25408,65 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "rpm": 2000, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "tpm": 800000, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -25759,6 +25875,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_audio_token": 7e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a3cfb300ea6..cc828e126ad 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23514,6 +23514,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "vertex_ai/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -25351,6 +25408,65 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "rpm": 2000, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "tpm": 800000, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -25759,6 +25875,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_audio_token": 7e-07, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 0e1c832ebf5..0ccb05c67a3 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -4200,6 +4200,51 @@ def test_generic_cost_per_token_gemini_37_flash(_local_model_cost_map): assert completion_cost == pytest.approx(0.001875) +GEMINI_38_FLASH_LAUNCH_PRICING = [ + ("gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08), + ("gemini/gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08), + ("vertex_ai/gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08), +] + + +@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_38_FLASH_LAUNCH_PRICING) +def test_gemini_38_flash_launch_pricing(model, input_cost, output_cost, cache_read_cost, _local_model_cost_map): + model_cost_map = litellm.model_cost[model] + assert model_cost_map["input_cost_per_token"] == input_cost + assert model_cost_map["output_cost_per_token"] == output_cost + assert model_cost_map["output_cost_per_reasoning_token"] == output_cost + assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost + assert model_cost_map["mode"] == "chat" + assert model_cost_map["supports_reasoning"] is True + assert model_cost_map["supports_function_calling"] is True + assert model_cost_map["max_input_tokens"] == 1048576 + + +def test_gemini_38_flash_matches_37_flash_promotional_pricing(_local_model_cost_map): + for prefix in ("", "gemini/", "vertex_ai/"): + assert litellm.model_cost[f"{prefix}gemini-3.8-flash"] == litellm.model_cost[f"{prefix}gemini-3.7-flash"] + + +def test_generic_cost_per_token_gemini_38_flash(_local_model_cost_map): + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=200, + text_tokens=300, + ), + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=1000), + ) + prompt_cost, completion_cost = generic_cost_per_token( + model="gemini-3.8-flash", + usage=usage, + custom_llm_provider="gemini", + ) + assert prompt_cost == pytest.approx(0.00075) + assert completion_cost == pytest.approx(0.001875) + + def test_grok_46_launch_pricing(_local_model_cost_map): model_cost_map = litellm.model_cost["xai/grok-4.6"] assert model_cost_map["input_cost_per_token"] == 2e-06 diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 8c1de12e7d9..4679b978f78 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -1096,10 +1096,13 @@ def test_natively_signed_parallel_turn_never_carries_a_placeholder(model): "gemini-3.5-flash", "gemini-3.6-flash", "gemini-3.7-flash", + "gemini-3.8-flash", "vertex_ai/gemini-3.5-flash", "vertex_ai/gemini-3.7-flash", + "vertex_ai/gemini-3.8-flash", "gemini/gemini-3.5-flash", "gemini/gemini-3.7-flash", + "gemini/gemini-3.8-flash", ], ) def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model): diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 521e91daded..0790b41c349 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4655,6 +4655,7 @@ GEMINI_4096_CACHE_MIN_MODELS: Final = tuple( "gemini-3.5-flash", "gemini-3.6-flash", "gemini-3.7-flash", + "gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3.1-pro-preview-customtools", ) From b76127774059d577229bbc9f74b3bf1b9fef812c Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 15:09:49 +0000 Subject: [PATCH 30/67] fix(models): drop inherited retirement dates from azure/us-gov entries pending a Government schedule source Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 ---- model_prices_and_context_window.json | 4 ---- 2 files changed, 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 87d348f4752..63b88c2a7b4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -58055,7 +58055,6 @@ "azure/us-gov/gpt-5.1": { "cache_read_input_token_cost": 1.71875e-07, "default_reasoning_effort": "none", - "deprecation_date": "2027-05-15", "input_cost_per_token": 1.71875e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -58090,7 +58089,6 @@ }, "azure/us-gov/o3-mini": { "cache_read_input_token_cost": 7.57e-07, - "deprecation_date": "2026-10-01", "input_cost_per_token": 1.513e-06, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -58105,7 +58103,6 @@ "supports_vision": false }, "azure/us-gov/text-embedding-3-large": { - "deprecation_date": "2028-02-09", "input_cost_per_token": 1.63e-07, "litellm_provider": "azure", "max_input_tokens": 8191, @@ -58114,7 +58111,6 @@ "output_cost_per_token": 0.0 }, "azure/us-gov/text-embedding-3-small": { - "deprecation_date": "2028-02-09", "input_cost_per_token": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 8191, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 87d348f4752..63b88c2a7b4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -58055,7 +58055,6 @@ "azure/us-gov/gpt-5.1": { "cache_read_input_token_cost": 1.71875e-07, "default_reasoning_effort": "none", - "deprecation_date": "2027-05-15", "input_cost_per_token": 1.71875e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -58090,7 +58089,6 @@ }, "azure/us-gov/o3-mini": { "cache_read_input_token_cost": 7.57e-07, - "deprecation_date": "2026-10-01", "input_cost_per_token": 1.513e-06, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -58105,7 +58103,6 @@ "supports_vision": false }, "azure/us-gov/text-embedding-3-large": { - "deprecation_date": "2028-02-09", "input_cost_per_token": 1.63e-07, "litellm_provider": "azure", "max_input_tokens": 8191, @@ -58114,7 +58111,6 @@ "output_cost_per_token": 0.0 }, "azure/us-gov/text-embedding-3-small": { - "deprecation_date": "2028-02-09", "input_cost_per_token": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 8191, From 07cf9dc46f5a4fd3b506f89cc77744f723eff190 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 08:10:52 -0700 Subject: [PATCH 31/67] feat(models): add Azure DeepSeek V4 Flash 0731 --- .../model_prices_and_context_window_backup.json | 16 ++++++++++++++++ model_prices_and_context_window.json | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a3cfb300ea6..893ac49f5c7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -10171,6 +10171,22 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-12-03", + "input_cost_per_token": 1.9e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.1e-07, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "azure_ai/deepseek-v4-flash-0731": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-12-03", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a3cfb300ea6..893ac49f5c7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -10171,6 +10171,22 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-12-03", + "input_cost_per_token": 1.9e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.1e-07, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "azure_ai/deepseek-v4-flash-0731": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-12-03", From da23e0241dc82649ff56f2e73e6e156c4e098129 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 2 Sep 2026 15:28:53 +0000 Subject: [PATCH 32/67] fix(models): add cloudflare whisper transcription pricing and pin govcloud pricing tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 20 ++ model_prices_and_context_window.json | 20 ++ .../test_bedrock_usgov_pricing.py | 200 +++++++++++++++--- ...st_cloudflare_workers_ai_model_metadata.py | 16 ++ 4 files changed, 230 insertions(+), 26 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 63b88c2a7b4..30621a17df3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -58117,5 +58117,25 @@ "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/openai/whisper": { + "input_cost_per_second": 7.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "cloudflare/@cf/openai/whisper-large-v3-turbo": { + "input_cost_per_second": 8.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper-large-v3-turbo/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 63b88c2a7b4..30621a17df3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -58117,5 +58117,25 @@ "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/openai/whisper": { + "input_cost_per_second": 7.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "cloudflare/@cf/openai/whisper-large-v3-turbo": { + "input_cost_per_second": 8.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper-large-v3-turbo/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] } } diff --git a/tests/test_litellm/test_bedrock_usgov_pricing.py b/tests/test_litellm/test_bedrock_usgov_pricing.py index 6b3312b5cc4..f9e8fd4c46c 100644 --- a/tests/test_litellm/test_bedrock_usgov_pricing.py +++ b/tests/test_litellm/test_bedrock_usgov_pricing.py @@ -26,9 +26,7 @@ import pytest @pytest.fixture(scope="module") def model_data(): - json_path = os.path.join( - os.path.dirname(__file__), "../../model_prices_and_context_window.json" - ) + json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json") with open(json_path) as f: return json.load(f) @@ -51,21 +49,14 @@ def test_usgov_sonnet_4_5_pricing(model_data, model_key): info = model_data[model_key] assert info["input_cost_per_token"] == 3.6e-06, ( - f"{model_key}: input_cost_per_token should be $3.60/MTok " - f"(got {info['input_cost_per_token']})" + f"{model_key}: input_cost_per_token should be $3.60/MTok (got {info['input_cost_per_token']})" ) - assert ( - info["output_cost_per_token"] == 1.8e-05 - ), f"{model_key}: output_cost_per_token should be $18.00/MTok" - assert ( - info["cache_creation_input_token_cost"] == 4.5e-06 - ), f"{model_key}: 5m cache write should be $4.50/MTok" - assert ( - info["cache_creation_input_token_cost_above_1hr"] == 7.2e-06 - ), f"{model_key}: 1h cache write should be $7.20/MTok" - assert ( - info["cache_read_input_token_cost"] == 3.6e-07 - ), f"{model_key}: cache read should be $0.36/MTok" + assert info["output_cost_per_token"] == 1.8e-05, f"{model_key}: output_cost_per_token should be $18.00/MTok" + assert info["cache_creation_input_token_cost"] == 4.5e-06, f"{model_key}: 5m cache write should be $4.50/MTok" + assert info["cache_creation_input_token_cost_above_1hr"] == 7.2e-06, ( + f"{model_key}: 1h cache write should be $7.20/MTok" + ) + assert info["cache_read_input_token_cost"] == 3.6e-07, f"{model_key}: cache read should be $0.36/MTok" def test_usgov_carries_20_percent_premium_over_global(model_data): @@ -84,9 +75,7 @@ def test_usgov_carries_20_percent_premium_over_global(model_data): "cache_read_input_token_cost", ): ratio = usgov_info[field] / global_info[field] - assert ( - abs(ratio - 1.2) < 1e-9 - ), f"{field}: us-gov / global ratio is {ratio}, expected 1.2" + assert abs(ratio - 1.2) < 1e-9, f"{field}: us-gov / global ratio is {ratio}, expected 1.2" # The us-gov.anthropic.* cross-region inference profile is the only us-gov @@ -112,9 +101,7 @@ def test_usgov_cross_region_above_200k_carries_gov_premium(model_data, field, ex """ info = model_data[USGOV_CROSS_REGION_KEY] assert field in info, f"{USGOV_CROSS_REGION_KEY}: missing field {field}" - assert ( - info[field] == expected - ), f"{USGOV_CROSS_REGION_KEY}: {field} should be {expected} (got {info[field]})" + assert info[field] == expected, f"{USGOV_CROSS_REGION_KEY}: {field} should be {expected} (got {info[field]})" def test_usgov_cross_region_above_200k_ratio_to_global(model_data): @@ -127,6 +114,167 @@ def test_usgov_cross_region_above_200k_ratio_to_global(model_data): usgov_info = model_data[USGOV_CROSS_REGION_KEY] for field in EXPECTED_USGOV_ABOVE_200K: ratio = usgov_info[field] / global_info[field] - assert ( - abs(ratio - 1.2) < 1e-9 - ), f"{field}: us-gov / global ratio is {ratio}, expected 1.2" + assert abs(ratio - 1.2) < 1e-9, f"{field}: us-gov / global ratio is {ratio}, expected 1.2" + + +CLAUDE_GOV_EXPECTED = { + "anthropic.claude-sonnet-5": { + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 1.2e-05, + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + }, + "anthropic.claude-opus-4-8": { + "input_cost_per_token": 6e-06, + "output_cost_per_token": 3e-05, + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + }, +} + + +@pytest.mark.parametrize("base_key", CLAUDE_GOV_EXPECTED) +@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"]) +def test_usgov_claude_sonnet5_opus48_pricing(model_data, region, base_key): + """Sonnet 5 and Opus 4.8 gov entries must match the rates AWS publishes + for both GovCloud regions on the Bedrock pricing page (1.2x global). + """ + gov_key = f"bedrock/{region}/{base_key}" + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + for field, expected in CLAUDE_GOV_EXPECTED[base_key].items(): + assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})" + ratio = info[field] / model_data[base_key][field] + assert abs(ratio - 1.2) < 1e-9, f"{gov_key}: {field} gov/global ratio is {ratio}, expected 1.2" + + +CONVERSE_GOV_EXPECTED = { + "nvidia.nemotron-nano-3-30b": (7.2e-08, 2.88e-07), + "nvidia.nemotron-nano-12b-v2": (2.4e-07, 7.2e-07), + "nvidia.nemotron-super-3-120b": (1.8e-07, 7.8e-07), + "openai.gpt-oss-20b-1:0": (8.4e-08, 3.6e-07), + "openai.gpt-oss-120b-1:0": (1.8e-07, 7.2e-07), +} + + +@pytest.mark.parametrize("base_key", CONVERSE_GOV_EXPECTED) +@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"]) +def test_usgov_converse_model_pricing(model_data, region, base_key): + """Nemotron and gpt-oss gov entries must match the AWS Bedrock offer file, + which prices both GovCloud regions identically at 1.2x commercial. + """ + gov_key = f"bedrock/{region}/{base_key}" + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + expected_input, expected_output = CONVERSE_GOV_EXPECTED[base_key] + assert info["input_cost_per_token"] == expected_input + assert info["output_cost_per_token"] == expected_output + assert info["litellm_provider"] == "bedrock" + base = model_data[base_key] + assert abs(info["input_cost_per_token"] / base["input_cost_per_token"] - 1.2) < 1e-9 + assert abs(info["output_cost_per_token"] / base["output_cost_per_token"] - 1.2) < 1e-9 + + +def test_usgov_west_llama3_8b_output_price_fixed(model_data): + """The us-gov-west-1 llama3-8b entry carried the 70B output rate ($2.65/MTok); + the AWS Bedrock offer file prices output at $0.60/MTok. AWS lists the model + in us-gov-west-1 only, so there is no east entry to check. + """ + info = model_data["bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0"] + assert info["input_cost_per_token"] == 3e-07 + assert info["output_cost_per_token"] == 6e-07 + + +MANTLE_GOV_TIERED_EXPECTED = { + "openai.gpt-5.6-luna": { + "input_cost_per_token": 2.64e-07, + "input_cost_per_token_above_272k_tokens": 5.28e-07, + "cache_creation_input_token_cost": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-07, + "cache_read_input_token_cost": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-08, + "output_cost_per_token": 1.584e-06, + "output_cost_per_token_above_272k_tokens": 2.376e-06, + }, + "openai.gpt-5.6-terra": { + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_above_272k_tokens": 5.28e-06, + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-06, + "cache_read_input_token_cost": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-07, + "output_cost_per_token": 1.584e-05, + "output_cost_per_token_above_272k_tokens": 2.376e-05, + }, +} + + +@pytest.mark.parametrize("model", MANTLE_GOV_TIERED_EXPECTED) +def test_usgov_west_mantle_terra_luna_pricing(model_data, model): + """Terra and Luna carry 1.2x commercial across every tier in the + us-gov-west-1 offer file; the us-gov-east-1 offer file has no SKUs for them. + """ + gov_key = f"bedrock_mantle/us-gov-west-1/{model}" + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + for field, expected in MANTLE_GOV_TIERED_EXPECTED[model].items(): + assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})" + assert info["litellm_provider"] == "bedrock_mantle" + assert f"bedrock_mantle/us-gov-east-1/{model}" not in model_data + + +@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"]) +def test_usgov_mantle_gpt_5_4_pricing_has_no_long_context_tier(model_data, region): + """gpt-5.4 gov rates come from the offer file, which publishes only the + standard tier in GovCloud: no long-context SKUs exist there, unlike commercial. + """ + gov_key = f"bedrock_mantle/{region}/openai.gpt-5.4" + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + assert info["input_cost_per_token"] == 3.3e-06 + assert info["cache_read_input_token_cost"] == 3.3e-07 + assert info["output_cost_per_token"] == 1.98e-05 + assert not any(field.endswith("_above_272k_tokens") for field in info) + + +def test_usgov_mantle_grok_4_3_west_only(model_data): + """grok-4.3 is priced in the us-gov-west-1 offer file only; the east offer + file carries grok-4.6 instead. + """ + info = model_data["bedrock_mantle/us-gov-west-1/xai.grok-4.3"] + assert info["input_cost_per_token"] == 1.5e-06 + assert info["output_cost_per_token"] == 3e-06 + assert info["cache_read_input_token_cost"] == 2.4e-07 + assert "bedrock_mantle/us-gov-east-1/xai.grok-4.3" not in model_data + + +AZURE_GOV_EXPECTED = { + "azure/us-gov/gpt-5.1": { + "input_cost_per_token": 1.71875e-06, + "cache_read_input_token_cost": 1.71875e-07, + "output_cost_per_token": 1.375e-05, + }, + "azure/us-gov/o3-mini": { + "input_cost_per_token": 1.513e-06, + "cache_read_input_token_cost": 7.57e-07, + "output_cost_per_token": 6.05e-06, + }, + "azure/us-gov/text-embedding-3-large": {"input_cost_per_token": 1.63e-07}, + "azure/us-gov/text-embedding-3-small": {"input_cost_per_token": 2.5e-08}, +} + + +@pytest.mark.parametrize("gov_key", AZURE_GOV_EXPECTED) +def test_azure_usgov_pricing(model_data, gov_key): + """Azure Government meters from the Azure retail prices API + (usgovvirginia/usgovarizona, serviceName 'Foundry Models'). No Government + retirement schedule is published, so these entries carry no deprecation_date. + """ + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + for field, expected in AZURE_GOV_EXPECTED[gov_key].items(): + assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})" + assert info["litellm_provider"] == "azure" + assert "deprecation_date" not in info diff --git a/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py b/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py index 9ca4515239a..e33bcfb8378 100644 --- a/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py +++ b/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py @@ -75,6 +75,22 @@ def test_additional_current_models_are_present(): assert entry["output_cost_per_token"] > 0 +@pytest.mark.parametrize( + "key, published_price_per_audio_minute", + [ + ("cloudflare/@cf/openai/whisper", 0.00045), + ("cloudflare/@cf/openai/whisper-large-v3-turbo", 0.00051), + ], +) +def test_whisper_transcription_pricing_is_stored_per_second(key, published_price_per_audio_minute): + entry = litellm.model_cost[key] + assert entry["litellm_provider"] == "cloudflare" + assert entry["mode"] == "audio_transcription" + assert entry["supported_endpoints"] == ["/v1/audio/transcriptions"] + assert entry["output_cost_per_second"] == 0.0 + assert entry["input_cost_per_second"] == pytest.approx(published_price_per_audio_minute / 60) + + def test_root_and_backup_have_identical_cloudflare_keys(): if not os.path.exists(ROOT_MAP): pytest.skip("root cost map only ships in source checkouts") From 2ce4e3f8a99e12efce9433640059d9fca7bfb448 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 08:32:49 -0700 Subject: [PATCH 33/67] fix(guardrails): run apply_guardrail-only providers in logging_only mode (#39297) * fix(guardrails): run apply_guardrail-only providers in logging_only mode A CustomGuardrail that implements only apply_guardrail inherited the CustomLogger no-op async_logging_hook, so mode: logging_only never scanned anything and never recorded guardrail_information. CustomGuardrail.async_logging_hook now routes the logged request and response through the call type's guardrail translation on copies and appends the verdict to standard_logging_object.guardrail_information. Resolves LIT-4876 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): keep logging_only scan copies inside the error boundary and return a fresh logging payload Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): cover embedding scan, native-hook bypass, and unmapped call type in logging_only Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 65 ++++++ .../integrations/test_custom_guardrail.py | 199 ++++++++++++++++++ 2 files changed, 264 insertions(+) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index e87ac9521ae..372c9bf6b91 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1,4 +1,5 @@ import contextvars +import copy import hashlib import os import secrets @@ -39,6 +40,7 @@ except ImportError: if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation dc: Final = DualCache() @@ -852,6 +854,69 @@ class CustomGuardrail(CustomLogger): return result + async def async_logging_hook( + self, + kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract + result: object, + call_type: str, + ) -> tuple[dict, object]: # mutable-ok: CustomLogger.async_logging_hook contract + """logging_only: run apply_guardrail on copies of the logged request/response and record the verdict.""" + from litellm.llms import get_guardrail_translation_mapping + + if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks: + return kwargs, result + try: + translation: Final = get_guardrail_translation_mapping(CallTypes(call_type))() + except ValueError: + verbose_logger.debug( + "Guardrail %s: no guardrail translation for call_type=%s, skipping logging_only scan", + self.guardrail_name, + call_type, + ) + return kwargs, result + litellm_params: Final = kwargs.get("litellm_params") or {} + scratch_metadata: Final = { + key: value + for key, value in (litellm_params.get("metadata") or {}).items() + if key != "standard_logging_guardrail_information" + } + try: + await self._scan_logged_call(kwargs, result, translation, scratch_metadata) + except Exception as e: + verbose_logger.warning("Guardrail %s: logging_only scan raised: %s", self.guardrail_name, e) + recorded: Final = scratch_metadata.get("standard_logging_guardrail_information") + standard_logging_object: Final = kwargs.get("standard_logging_object") + if not recorded or not isinstance(standard_logging_object, dict): + return kwargs, result + entries: Final = recorded if isinstance(recorded, list) else [recorded] + existing: Final = standard_logging_object.get("guardrail_information") or [] + return { + **kwargs, + "standard_logging_object": {**standard_logging_object, "guardrail_information": [*existing, *entries]}, + }, result + + async def _scan_logged_call( + self, + kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract + result: object, + translation: "BaseTranslation", + scratch_metadata: dict, # mutable-ok: apply_guardrail records its verdict into request metadata + ) -> None: + optional_params: Final = kwargs.get("optional_params") or {} + scratch_input: Final = copy.deepcopy(kwargs.get("messages") or kwargs.get("input")) + scratch_request: Final = { + "model": kwargs.get("model"), + "messages": scratch_input, + "input": scratch_input, + "tools": copy.deepcopy(optional_params.get("tools")), + "litellm_call_id": kwargs.get("litellm_call_id"), + "metadata": scratch_metadata, + } + await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self) + await translation.process_output_response( + response=copy.deepcopy(result), guardrail_to_apply=self, request_data=scratch_request + ) + def supports_scan_only_tool_results(self) -> bool: """Whether this guardrail can scan tool-result content. diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index d978eb48c12..7d70b9a8862 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -2237,3 +2237,202 @@ class TestRecordsOwnGuardrailInformation: ) assert _guardrail_entries(request_data) == [] + + +class _ApplyOnlyObserver(CustomGuardrail): + """Overrides only apply_guardrail, like panw_prisma_airs; inherits async_logging_hook.""" + + def __init__(self, block: bool = False): + from litellm.types.guardrails import GuardrailEventHooks + + super().__init__(guardrail_name="apply-only-observer", event_hook=GuardrailEventHooks.logging_only) + self.block = block + self.calls: list = [] + + @log_guardrail_information + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + from fastapi import HTTPException + + self.calls.append((input_type, list(inputs.get("texts") or []))) + if self.block: + raise HTTPException(status_code=400, detail={"error": "flagged"}) + return GenericGuardrailAPIInputs(texts=["[MASKED]" for _ in inputs.get("texts") or []]) + + +def _logged_call(messages: list | str) -> tuple[dict, object]: + from litellm.types.utils import Choices, Message, ModelResponse + + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="general kenobi"))]) + kwargs = { + "model": "gpt-5.4-mini", + "messages": messages, + "litellm_call_id": "call-1", + "litellm_params": {"metadata": {"user_api_key_user_id": "u1"}}, + "optional_params": {}, + "standard_logging_object": {"guardrail_information": None}, + } + return kwargs, response + + +class TestLoggingOnlyApplyGuardrail: + """LIT-4876 regression: a guardrail in mode logging_only that implements only + apply_guardrail must still run against the logged request and response and + record guardrail_information, instead of inheriting the CustomLogger no-op.""" + + @pytest.mark.asyncio + async def test_runs_apply_guardrail_observe_only_and_records_verdict(self): + guardrail = _ApplyOnlyObserver() + messages = [{"role": "user", "content": "hello there"}] + kwargs, response = _logged_call(messages) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])] + assert out_kwargs["messages"] == [{"role": "user", "content": "hello there"}] + assert out_response.choices[0].message.content == "general kenobi" + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_name"] for e in entries] == ["apply-only-observer", "apply-only-observer"] + assert {e["guardrail_mode"] for e in entries} == {"logging_only"} + assert {e["guardrail_status"] for e in entries} == {"success"} + assert "standard_logging_guardrail_information" not in kwargs["litellm_params"]["metadata"] + assert kwargs["standard_logging_object"] == {"guardrail_information": None} + + @pytest.mark.asyncio + async def test_appends_to_pre_call_verdicts_without_duplicating_them(self): + guardrail = _ApplyOnlyObserver() + kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) + pre_call_entry = {"guardrail_name": "pii-blocker", "guardrail_mode": "pre_call", "guardrail_status": "success"} + kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] = [pre_call_entry] + kwargs["standard_logging_object"]["guardrail_information"] = [pre_call_entry] + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_name"] for e in entries] == ["pii-blocker", "apply-only-observer", "apply-only-observer"] + assert kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] == [pre_call_entry] + + @pytest.mark.asyncio + async def test_request_copy_failure_is_swallowed(self): + import threading + + guardrail = _ApplyOnlyObserver() + kwargs, response = _logged_call([{"role": "user", "content": "hello there", "lock": threading.Lock()}]) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [] + assert out_kwargs is kwargs + assert out_response is response + + @pytest.mark.asyncio + async def test_block_verdict_is_recorded_without_raising(self): + guardrail = _ApplyOnlyObserver(block=True) + kwargs, response = _logged_call([{"role": "user", "content": "flagged content"}]) + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [("request", ["flagged content"])] + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["guardrail_intervened"] + + @pytest.mark.asyncio + async def test_call_type_without_translation_is_skipped(self): + guardrail = _ApplyOnlyObserver() + kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.amoderation.value) + + assert guardrail.calls == [] + assert out_kwargs["standard_logging_object"]["guardrail_information"] is None + + @pytest.mark.asyncio + async def test_aembedding_scans_logged_input(self): + from litellm.types.utils import EmbeddingResponse + + guardrail = _ApplyOnlyObserver() + kwargs, _ = _logged_call("hello there") + response = EmbeddingResponse(data=[{"embedding": [0.1], "index": 0, "object": "embedding"}]) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.aembedding.value) + + assert guardrail.calls == [("request", ["hello there"])] + assert out_kwargs["messages"] == "hello there" + assert out_response is response + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["success"] + + @pytest.mark.asyncio + async def test_native_lifecycle_hook_guardrail_is_left_alone(self): + class _NativeHooks(_ApplyOnlyObserver): + use_native_lifecycle_hooks = True + + guardrail = _NativeHooks() + kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [] + assert out_kwargs is kwargs + assert out_response is response + + @pytest.mark.asyncio + async def test_aresponses_scans_logged_messages_when_input_is_cleared(self): + from litellm.types.llms.openai import ResponsesAPIResponse + + guardrail = _ApplyOnlyObserver() + kwargs, _ = _logged_call([{"role": "user", "content": "hello there"}]) + kwargs["input"] = None + response = ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="gpt-5.4-mini", + object="response", + status="completed", + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "general kenobi"}], + } + ], + ) + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.aresponses.value) + + assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])] + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["success", "success"] + + @pytest.mark.asyncio + async def test_async_success_handler_records_verdict_in_standard_logging_object(self): + import datetime as dt + + from litellm.litellm_core_utils.litellm_logging import Logging + + guardrail = _ApplyOnlyObserver() + guardrail.default_on = True + messages = [{"role": "user", "content": "hello there"}] + _, response = _logged_call(messages) + logging_obj = Logging( + model="gpt-5.4-mini", + messages=messages, + stream=False, + call_type=CallTypes.acompletion.value, + start_time=dt.datetime.now(), + litellm_call_id="call-1", + function_id="fn-1", + dynamic_async_success_callbacks=[guardrail], + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {}}, optional_params={}, model="gpt-5.4-mini", custom_llm_provider="openai" + ) + + await logging_obj.async_success_handler( + result=response, start_time=dt.datetime.now(), end_time=dt.datetime.now() + ) + + assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])] + entries = logging_obj.model_call_details["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["success", "success"] From 69cd1bada6249889a9412155a6086faea65bfe6c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 08:46:28 -0700 Subject: [PATCH 34/67] test(gemini): compare gemini-3.8-flash to 3.7 flash field by field --- .../llm_cost_calc/test_llm_cost_calc_utils.py | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 0ccb05c67a3..9b3e60764e3 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -4220,9 +4220,44 @@ def test_gemini_38_flash_launch_pricing(model, input_cost, output_cost, cache_re assert model_cost_map["max_input_tokens"] == 1048576 -def test_gemini_38_flash_matches_37_flash_promotional_pricing(_local_model_cost_map): - for prefix in ("", "gemini/", "vertex_ai/"): - assert litellm.model_cost[f"{prefix}gemini-3.8-flash"] == litellm.model_cost[f"{prefix}gemini-3.7-flash"] +GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH = ( + "input_cost_per_token", + "output_cost_per_token", + "output_cost_per_reasoning_token", + "cache_read_input_token_cost", + "input_cost_per_token_batches", + "output_cost_per_token_batches", + "input_cost_per_token_flex", + "output_cost_per_token_flex", + "cache_read_input_token_cost_flex", + "input_cost_per_token_priority", + "output_cost_per_token_priority", + "cache_read_input_token_cost_priority", + "search_context_cost_per_query", + "google_maps_grounding_cost_per_query", + "prompt_cache_min_tokens", + "max_input_tokens", + "max_output_tokens", + "supports_reasoning", + "supports_function_calling", + "supports_prompt_caching", + "supports_vision", + "supports_pdf_input", + "supports_audio_input", + "supports_video_input", + "supports_response_schema", + "supports_tool_choice", + "supports_web_search", + "supports_url_context", +) + + +@pytest.mark.parametrize("prefix", ["", "gemini/", "vertex_ai/"]) +def test_gemini_38_flash_matches_37_flash_promotional_pricing(prefix, _local_model_cost_map): + new_model = litellm.model_cost[f"{prefix}gemini-3.8-flash"] + old_model = litellm.model_cost[f"{prefix}gemini-3.7-flash"] + for field in GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH: + assert new_model[field] == old_model[field], field def test_generic_cost_per_token_gemini_38_flash(_local_model_cost_map): From de80e3afe448237c69a535517ca88c12d079ee6d Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 2 Sep 2026 10:19:03 -0700 Subject: [PATCH 35/67] fix(helm): scale the classic chart's HPA out at the documented 60 percent CPU (#35975) * fix(helm): scale the classic chart's HPA out at the documented 60 percent CPU The litellm-helm chart shipped targetCPUUtilizationPercentage: 80, which is unexamined helm create scaffold rather than a chosen number. It arrived packaged with the stock minReplicas: 1, maxReplicas: 100, a commented-out targetMemoryUtilizationPercentage: 80, and the boilerplate "such as Minikube" comment, the same provenance as the 128Mi resource example this file just corrected. 60 is the documented recommendation. The mechanism behind it is scale-up lag: the chart's own startupProbe is failureThreshold: 30 times periodSeconds: 10, so a replica can take up to 300 seconds to become ready, and a pod added at 80 percent utilization arrives minutes after saturation. The memory target stays commented out on purpose. The prisma query engine's resident memory is a high-water mark that ratchets to the pod's worst-ever write and is never returned, so a memory-target HPA reads the largest write a pod ever did rather than what it is doing now, and replicas ratchet up without scaling back in. hpa_tests.yaml carried its second suite after a YAML document separator, and helm-unittest loads only the first document per file, so that suite never ran; an assertion planted in it still passed. Fold it into the one live suite and add coverage pinning the rendered CPU target, the absence of a memory metric by default, and that overrides still take effect. Bump the chart to 1.1.2, since rendered output changes for anyone running with autoscaling enabled. * fix(helm): bump litellm-helm to 1.1.3 after rebase onto 1.1.2 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- helm/litellm-helm/Chart.yaml | 2 +- helm/litellm-helm/tests/hpa_tests.yaml | 42 ++++++++++++++++++++++---- helm/litellm-helm/values.yaml | 11 ++++++- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/helm/litellm-helm/Chart.yaml b/helm/litellm-helm/Chart.yaml index 3959d85edf3..a3cb388ffc6 100644 --- a/helm/litellm-helm/Chart.yaml +++ b/helm/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.1.2 +version: 1.1.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/litellm-helm/tests/hpa_tests.yaml b/helm/litellm-helm/tests/hpa_tests.yaml index ec18c3591d3..cd062dd5971 100644 --- a/helm/litellm-helm/tests/hpa_tests.yaml +++ b/helm/litellm-helm/tests/hpa_tests.yaml @@ -1,4 +1,4 @@ -suite: "hpa with behavior" +suite: "hpa" templates: - hpa.yaml tests: @@ -23,14 +23,44 @@ tests: - equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 } - equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 } ---- -suite: "hpa without behavior" -templates: - - hpa.yaml -tests: - it: "does not render behavior when not set" set: autoscaling.enabled: true asserts: - isKind: { of: HorizontalPodAutoscaler } - isNull: { path: spec.behavior } + + - it: "scales on cpu at the documented 60 percent by default" + set: + autoscaling.enabled: true + asserts: + - isKind: { of: HorizontalPodAutoscaler } + - equal: { path: "spec.metrics[0].resource.name", value: cpu } + - equal: { path: "spec.metrics[0].resource.target.type", value: Utilization } + - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 } + + - it: "does not scale on memory by default" + set: + autoscaling.enabled: true + asserts: + - lengthEqual: { path: spec.metrics, count: 1 } + + - it: "honours an explicit cpu target override" + set: + autoscaling.enabled: true + autoscaling.targetCPUUtilizationPercentage: 75 + asserts: + - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 } + + - it: "renders a memory metric only when a memory target is set" + set: + autoscaling.enabled: true + autoscaling.targetMemoryUtilizationPercentage: 80 + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - equal: { path: "spec.metrics[1].resource.name", value: memory } + - equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 } + + - it: "renders no hpa when autoscaling is disabled" + asserts: + - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index f8df98de102..637be2322e3 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -200,7 +200,16 @@ autoscaling: enabled: false minReplicas: 1 maxReplicas: 100 - targetCPUUtilizationPercentage: 80 + # 60 is the documented recommendation. See "Recommended Machine Specifications" + # in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe + # above only after up to failureThreshold x periodSeconds = 300 seconds, so a target + # high enough to trip near saturation adds capacity minutes after it was needed. + targetCPUUtilizationPercentage: 60 + # Deliberately left unset rather than given a value. The prisma query engine's + # resident memory is a high-water mark that ratchets to the pod's worst-ever write + # and is never returned, so a memory target reads the largest write a pod ever did + # rather than what it is doing now, and replicas ratchet up without scaling back in. + # Memory is a floor to provision under 'resources', not a signal to scale on. # targetMemoryUtilizationPercentage: 80 # behavior: {} From 7a35c34303e944f68e69299346ec04371b5f59c7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:30:14 -0700 Subject: [PATCH 36/67] fix(models): add the us-gov. geo inference profile keys for Claude Sonnet 5 and Opus 4.8 --- ...odel_prices_and_context_window_backup.json | 64 +++++++++++++++++++ model_prices_and_context_window.json | 64 +++++++++++++++++++ .../test_bedrock_usgov_pricing.py | 19 ++++-- 3 files changed, 142 insertions(+), 5 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 30621a17df3..28c503966f3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -42008,6 +42008,70 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024 }, + "us-gov.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us-gov.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 30621a17df3..28c503966f3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -42008,6 +42008,70 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024 }, + "us-gov.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us-gov.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, diff --git a/tests/test_litellm/test_bedrock_usgov_pricing.py b/tests/test_litellm/test_bedrock_usgov_pricing.py index f9e8fd4c46c..f7d95ecda01 100644 --- a/tests/test_litellm/test_bedrock_usgov_pricing.py +++ b/tests/test_litellm/test_bedrock_usgov_pricing.py @@ -135,15 +135,24 @@ CLAUDE_GOV_EXPECTED = { } +USGOV_CLAUDE_KEY_TEMPLATES = { + "bedrock/us-gov-east-1/{base_key}": "bedrock", + "bedrock/us-gov-west-1/{base_key}": "bedrock", + "us-gov.{base_key}": "bedrock_converse", +} + + @pytest.mark.parametrize("base_key", CLAUDE_GOV_EXPECTED) -@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"]) -def test_usgov_claude_sonnet5_opus48_pricing(model_data, region, base_key): - """Sonnet 5 and Opus 4.8 gov entries must match the rates AWS publishes - for both GovCloud regions on the Bedrock pricing page (1.2x global). +@pytest.mark.parametrize("key_template,expected_provider", USGOV_CLAUDE_KEY_TEMPLATES.items()) +def test_usgov_claude_sonnet5_opus48_pricing(model_data, key_template, expected_provider, base_key): + """Sonnet 5 and Opus 4.8 gov entries, both in-region keys and the us-gov. + geo inference profile the model cards list for GovCloud, must match the + rates AWS publishes on the Bedrock pricing page (1.2x global). """ - gov_key = f"bedrock/{region}/{base_key}" + gov_key = key_template.format(base_key=base_key) assert gov_key in model_data, f"Missing model entry: {gov_key}" info = model_data[gov_key] + assert info["litellm_provider"] == expected_provider for field, expected in CLAUDE_GOV_EXPECTED[base_key].items(): assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})" ratio = info[field] / model_data[base_key][field] From ffc0a8e428a4d8af7e5b130bddb4f0f9cf0cb229 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:51:17 -0700 Subject: [PATCH 37/67] fix: run access group key sync UPDATEs on the writer, not the read replica (#39128) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../access_group_key_sync.py | 6 +- .../test_access_group_key_sync.py | 57 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/proxy/management_helpers/test_access_group_key_sync.py diff --git a/litellm/proxy/management_helpers/access_group_key_sync.py b/litellm/proxy/management_helpers/access_group_key_sync.py index 5d43cb29978..c9f93fae0d9 100644 --- a/litellm/proxy/management_helpers/access_group_key_sync.py +++ b/litellm/proxy/management_helpers/access_group_key_sync.py @@ -38,6 +38,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_checks import ( _delete_cache_access_object, # pyright: ignore[reportPrivateUsage] # the access-group endpoints reach for this same cache primitive ) +from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient from litellm.repositories.table_repositories import AccessGroupRepository @@ -72,8 +73,9 @@ _REPOINT_KEY_SQL: Final = ( def _raw_executor(prisma_client: object) -> _RawExecutor: - """Narrow the untyped Prisma client down to the raw-query call this module makes.""" - return AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client + """Narrow the untyped Prisma client down to the raw-query call this module makes, pinned to the writer.""" + db: Final = AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client + return WriterPinnedClient(db).db # pyright: ignore[reportAny, reportReturnType] # untyped Prisma client behind the pin async def _invalidate_access_group_cache(access_group_id: str) -> None: diff --git a/tests/test_litellm/proxy/management_helpers/test_access_group_key_sync.py b/tests/test_litellm/proxy/management_helpers/test_access_group_key_sync.py new file mode 100644 index 00000000000..60c36e33e09 --- /dev/null +++ b/tests/test_litellm/proxy/management_helpers/test_access_group_key_sync.py @@ -0,0 +1,57 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.proxy.db.prisma_client import PrismaWrapper +from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper +from litellm.proxy.management_helpers.access_group_key_sync import ( + sync_key_access_group_membership, + sync_key_regeneration_access_group_membership, +) + + +def _routed_prisma_client(): + writer_inner = MagicMock(name="writer_prisma") + reader_inner = MagicMock(name="reader_prisma") + writer_inner.query_raw = AsyncMock(return_value=[]) + reader_inner.query_raw = AsyncMock(return_value=[]) + writer = PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False) + reader = PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False) + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + return SimpleNamespace(db=routing), writer_inner, reader_inner + + +@pytest.mark.asyncio +async def test_regeneration_repoint_update_runs_on_the_writer(): + prisma_client, writer_inner, reader_inner = _routed_prisma_client() + + await sync_key_regeneration_access_group_membership( + prisma_client=prisma_client, + previous_key_token="old-token", + new_key_token="new-token", + data=None, + existing_key_row=MagicMock(), + ) + + writer_inner.query_raw.assert_awaited_once() + assert writer_inner.query_raw.await_args.args[0].startswith('UPDATE "LiteLLM_AccessGroupTable"') + reader_inner.query_raw.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_membership_attach_and_detach_updates_run_on_the_writer(): + prisma_client, writer_inner, reader_inner = _routed_prisma_client() + + await sync_key_access_group_membership( + prisma_client=prisma_client, + key_token="token", + previous_access_group_ids=["ag-old"], + updated_access_group_ids=["ag-new"], + ) + + assert writer_inner.query_raw.await_count == 2 + assert all( + call.args[0].startswith('UPDATE "LiteLLM_AccessGroupTable"') for call in writer_inner.query_raw.await_args_list + ) + reader_inner.query_raw.assert_not_awaited() From a9d3a0746c582de8910d0a7078489ac961e436e8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:59:11 -0700 Subject: [PATCH 38/67] fix(models): price Azure DeepSeek V4 Flash 0731 from its own meters under the catalog id --- ...odel_prices_and_context_window_backup.json | 22 +++---------------- model_prices_and_context_window.json | 22 +++---------------- 2 files changed, 6 insertions(+), 38 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 893ac49f5c7..0af5a89742e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -10172,31 +10172,15 @@ "supports_tool_choice": true }, "azure_ai/DeepSeek-V4-Flash-0731": { - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, + "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.1e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "azure_ai/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 2.8e-08, - "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 5.1e-07, + "output_cost_per_token": 1.32e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_prompt_caching": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 893ac49f5c7..0af5a89742e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -10172,31 +10172,15 @@ "supports_tool_choice": true }, "azure_ai/DeepSeek-V4-Flash-0731": { - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, + "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.1e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "azure_ai/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 2.8e-08, - "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 5.1e-07, + "output_cost_per_token": 1.32e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_prompt_caching": true, From e0be9a35e6838505b5a2ae2ecf93c01579a02f56 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:02:43 -0700 Subject: [PATCH 39/67] fix(deps): raise the pypdf floor to 6.16.1 for three new advisories GHSA-jp53-mhqp-8xcg (fixed in 6.16.0), GHSA-23w6-3w8w-8484 and GHSA-763m-79hh-57f2 (fixed in 6.16.1) flag pypdf 6.15.0 in uv.lock and keep osv-scan red alongside the tornado advisories. The proxy-runtime extra now requires pypdf>=6.16.1 and the lock resolves 6.16.2. --- pyproject.toml | 2 +- uv.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d0f14722acd..60162544612 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,7 +161,7 @@ proxy-runtime = [ "mangum>=0.17.0,<1.0", "azure-ai-contentsafety>=1.0.0,<2.0", "azure-storage-file-datalake>=12.20.0,<13.0", - "pypdf>=6.12.0,<7.0", + "pypdf>=6.16.1,<7.0", "llm-sandbox>=0.3.39,<1.0", "detect-secrets>=1.5.0,<2.0", ] diff --git a/uv.lock b/uv.lock index 8bac024d49e..aa59ff7b229 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-29T20:52:40.322465Z" +exclude-newer = "2026-08-30T17:51:25.171404Z" exclude-newer-span = "P3D" [manifest] @@ -4552,7 +4552,7 @@ requires-dist = [ { name = "pydantic-settings", specifier = ">=2.14.1,<3.0" }, { name = "pyjwt", marker = "extra == 'proxy'", specifier = ">=2.13.0,<3.0" }, { name = "pynacl", marker = "extra == 'proxy'", specifier = ">=1.6.2,<2.0" }, - { name = "pypdf", marker = "extra == 'proxy-runtime'", specifier = ">=6.12.0,<7.0" }, + { name = "pypdf", marker = "extra == 'proxy-runtime'", specifier = ">=6.16.1,<7.0" }, { name = "pyroscope-io", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.8.16,<1.0" }, { name = "python-dotenv", specifier = ">=1.0.0,<2.0" }, { name = "python-multipart", marker = "extra == 'proxy'", specifier = ">=0.0.27,<1.0" }, @@ -7564,14 +7564,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.15.0" +version = "6.16.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/17/ee75a92718ec7212de831e71454d702225aa5e474a805cce169806044453/pypdf-6.15.0.tar.gz", hash = "sha256:d39c4d955a76409284a905e2d65b40076d77ab76129e0faaeeb6612403ecfc79", size = 6993794, upload-time = "2026-08-06T13:06:49.929Z" } +sdist = { url = "https://files.pythonhosted.org/packages/44/66/54212e75406afd9f3e933d0dda23072f6aecc55c5a273077dc2e0b028b23/pypdf-6.16.2.tar.gz", hash = "sha256:595647f6191de6f402cfde1d0c455d6cbccbd509aac32b34783009c032de5d6e", size = 7008996, upload-time = "2026-08-23T13:50:07.135Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/72/ce3067ac31e214a66388159f8462ddb8c13dd00170f24d555a1f1ae8ee91/pypdf-6.15.0-py3-none-any.whl", hash = "sha256:14e001d6504822cb1ca9c7ed9a69bccb320f59b320730f55af804361abe4d5ee", size = 378123, upload-time = "2026-08-06T13:06:47.709Z" }, + { url = "https://files.pythonhosted.org/packages/13/f1/a2da3b55acd4ab737bf728c97edaaed5ec1d3c1236acb639dcdfa97e42c7/pypdf-6.16.2-py3-none-any.whl", hash = "sha256:c8b09a59399062fb45a1b8156c18a787a10a3dae03ac9674397a226712c94604", size = 385060, upload-time = "2026-08-23T13:50:05.349Z" }, ] [[package]] From a677242d6f07af683b9c146133287f07b4e1459c Mon Sep 17 00:00:00 2001 From: Ali Ahmed <128928915+QuantumBreakz@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:07:13 +0500 Subject: [PATCH 40/67] fix(headroom): stop re-compressing retrieved CCR content in client tool loops (#38591) When the headroom_retrieve tool is exposed to a client that runs its own tool-execution loop (the LiteLLM MCP gateway path), the client executes the retrieve call and sends the recovered original content back as a tool result on the next turn. The guardrail then compressed that row again, and because CCR is content-addressed it collapsed back to the exact same hash it was just retrieved from. The model never saw the expansion and the agent looped. Hold tool-result rows that carry headroom_retrieve output back from the compression service, the same way the live turn and trailing tool exchange are already protected, so the expansion survives. Retrieve calls are matched by the direct headroom_retrieve name and the mcp____headroom_retrieve gateway name. Because a long gateway name is truncated past 64 chars in the OpenAI-translated view the guardrail scans, the pairing also falls back to the tool-call id read from the request's own untranslated messages, which is never truncated. Fixes #38558 --- .../guardrail_hooks/headroom/headroom.py | 120 ++++++++++++- .../guardrail_hooks/test_headroom.py | 158 ++++++++++++++++++ 2 files changed, 274 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index d8c8c2f4974..fc881a60f43 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, TypeGuard import httpx from fastapi import HTTPException from httpx import Response as HttpxResponse +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -52,6 +53,10 @@ BYPASS_HEADER: Final = "x-headroom-bypass" HEADROOM_RETRIEVE_TOOL_NAME: Final = "headroom_retrieve" _HASH_PATTERN: Final = re.compile(r"hash=([a-f0-9]{24})") _HASH_CACHE_TTL_SECONDS: Final = 15 * 60 +# Narrows the base class's bare-dict ``request_data`` at the boundary so its +# untranslated messages can be read with concrete types (values pass through by +# reference, so this is a shallow top-level reconstruction). +_REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object]) def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip @@ -116,16 +121,119 @@ def _restore_content_shapes( return restored -def _protected_indices(messages: Sequence[Mapping[str, object]]) -> frozenset[int]: +def _tool_call_name(tool_call: Mapping[str, object]) -> str | None: + function: Final = tool_call.get("function") + if not _is_str_object_dict(function): + return None + name: Final = function.get("name") + return name if isinstance(name, str) else None + + +def _is_retrieve_tool_name(name: str | None) -> bool: + """Match the retrieve tool whether called directly or via the MCP gateway. + + Server-side the tool is ``headroom_retrieve``; exposed through LiteLLM's MCP + gateway a client calls it as ``mcp____headroom_retrieve``. + """ + return name is not None and ( + name == HEADROOM_RETRIEVE_TOOL_NAME or name.endswith(f"__{HEADROOM_RETRIEVE_TOOL_NAME}") + ) + + +def _retrieve_call_ids_in_message(message: Mapping[str, object]) -> frozenset[str]: + if message.get("role") != "assistant": + return frozenset() + tool_calls: Final = message.get("tool_calls") + if not _is_object_list(tool_calls): + return frozenset() + return frozenset( + str(tool_call["id"]) + for tool_call in tool_calls + if _is_str_object_dict(tool_call) and tool_call.get("id") and _is_retrieve_tool_name(_tool_call_name(tool_call)) + ) + + +def _anthropic_tool_use_retrieve_id(block: object) -> str | None: + if not _is_str_object_dict(block) or block.get("type") != "tool_use": + return None + name: Final = block.get("name") + call_id: Final = block.get("id") + if isinstance(name, str) and call_id is not None and _is_retrieve_tool_name(name): + return str(call_id) + return None + + +def _anthropic_retrieve_ids_in_message(message: Mapping[str, object]) -> frozenset[str]: + content: Final = message.get("content") + if not _is_object_list(content): + return frozenset() + return frozenset(call_id for block in content if (call_id := _anthropic_tool_use_retrieve_id(block)) is not None) + + +def _raw_retrieve_call_ids(messages: object) -> frozenset[str]: + """Retrieve-tool call ids read from the request's own, untranslated messages. + + The guardrail otherwise scans an OpenAI-translated view where a tool name + over 64 chars is truncated to ``{prefix}_{hash}``, which drops the + ``__headroom_retrieve`` suffix a long ``mcp____`` prefix pushes past + the limit. Tool-call ids are never truncated, so pairing the tool result to + an id read from the original request keeps the match intact. Both wire + shapes are handled: OpenAI ``tool_calls`` and Anthropic ``tool_use`` blocks. + """ + if not _is_object_list(messages): + return frozenset() + return frozenset( + call_id + for message in messages + if _is_str_object_dict(message) + for call_id in _retrieve_call_ids_in_message(message) | _anthropic_retrieve_ids_in_message(message) + ) + + +def _retrieval_result_indices( + messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset() +) -> frozenset[int]: + """Indices of tool-result rows that carry ``headroom_retrieve`` output. + + When the retrieve tool is exposed to a client that runs its own tool loop + (the LiteLLM MCP gateway path), the client executes the call and sends the + recovered original content back as a tool result on the next turn. That + content is exactly what a prior compression stubbed, so compressing it again + re-derives the identical content hash: a no-op that strands the model on the + marker and loops the agent. Hold those rows back so the expansion survives. + + ``extra_retrieve_call_ids`` carries ids recovered from the untruncated + request so the pairing survives tool-name truncation (see + ``_raw_retrieve_call_ids``). + """ + retrieve_call_ids: Final = extra_retrieve_call_ids | frozenset( + call_id for message in messages for call_id in _retrieve_call_ids_in_message(message) + ) + if not retrieve_call_ids: + return frozenset() + return frozenset( + index + for index, message in enumerate(messages) + if message.get("role") in ("tool", "function") and str(message.get("tool_call_id")) in retrieve_call_ids + ) + + +def _protected_indices( + messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset() +) -> frozenset[int]: """Indices headroom must not send to the compression service. ``get_protected_indices`` is litellm's own compression policy: the system - rows, the last user row, the last assistant row. It is expanded over whole + rows, the last user row, the last assistant row. Rows carrying just-retrieved + ``headroom_retrieve`` output are added so re-compression can't collapse them + back to the marker they were expanded from. The union is expanded over whole tool exchanges the way ``compress()`` expands it, so a protected assistant tool call cannot end up answered by a marker standing in for the result the model just asked for. """ - protected: Final = frozenset(get_protected_indices(messages)) + protected: Final = frozenset(get_protected_indices(messages)) | _retrieval_result_indices( + messages, extra_retrieve_call_ids + ) return protected | frozenset( index for group in group_tool_exchanges(messages) @@ -634,7 +742,11 @@ class HeadroomGuardrail(CustomGuardrail): # /v1/compress grows a field for sending the live turn as the retrieval # query without compressing it: query-aware compression reads the newest # user message, so it is withheld here at some cost to history ranking. - protected_indices: Final = _protected_indices(messages) + # request_data is a bare dict on the base signature; narrow it before + # reading the untranslated messages so long tool names can be recovered. + raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages") + raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages) + protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids) compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices] if not compressible: return inputs diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index 1fbc975e40a..c04fb7b30ec 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -2199,6 +2199,164 @@ async def test_history_is_still_compressed(guardrail: HeadroomGuardrail): assert has_headroom_retrieve_tool(result.get("tools") or []) +# --------------------------------------------------------------------------- +# #38558: a client that runs its own tool loop (e.g. Claude Code via the MCP +# gateway) executes headroom_retrieve and echoes the recovered original content +# back as a tool result. Compressing that row re-derives the same content hash +# it was just retrieved from -- the marker returns and the agent loops. The +# retrieved row must be held back from the compression service. +# --------------------------------------------------------------------------- + +RETRIEVE_ECHO_MESSAGES = [ + {"role": "system", "content": "You are Claude Code. " + "S" * 5000}, + {"role": "user", "content": "H" * 5000}, + { + "role": "assistant", + "content": "Expanding the marker.", + "tool_calls": [ + { + "id": "hr_1", + "type": "function", + "function": { + "name": "mcp__headroom__headroom_retrieve", + "arguments": '{"hash": "b573993006976af767214fac"}', + }, + } + ], + }, + {"role": "tool", "tool_call_id": "hr_1", "content": "RETRIEVED BODY " + "R" * 5000}, + {"role": "assistant", "content": "Older answer. " + "O" * 5000}, + {"role": "user", "content": "now summarize the description"}, +] + + +@pytest.mark.asyncio +async def test_retrieved_content_is_never_recompressed(guardrail: HeadroomGuardrail): + """The tool result carrying headroom_retrieve output is held back, so it can + never collapse back to the hash it was just retrieved from.""" + wire, result = await _wire_and_result(guardrail, RETRIEVE_ECHO_MESSAGES) + + assert not any(row.get("tool_call_id") == "hr_1" for row in wire) + assert not any("RETRIEVED BODY" in json.dumps(row) for row in wire) + # Reaches the model byte-identical, so no marker stands in for the expansion. + assert result["structured_messages"][3] == RETRIEVE_ECHO_MESSAGES[3] + # Negative control: unrelated history is still compressed, not a no-op. + assert any(row.get("content") == "H" * 5000 for row in wire) + + +@pytest.mark.asyncio +async def test_retrieved_content_guard_matches_direct_tool_name(guardrail: HeadroomGuardrail): + """Server-side the tool is named headroom_retrieve (no MCP prefix); its + result must be protected the same way.""" + messages = [ + {"role": "system", "content": "sys " + "S" * 5000}, + {"role": "user", "content": "H" * 5000}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "hr_direct", + "type": "function", + "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME, "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "hr_direct", "content": "RETRIEVED BODY " + "R" * 5000}, + {"role": "assistant", "content": "Older. " + "O" * 5000}, + {"role": "user", "content": "summarize"}, + ] + wire, result = await _wire_and_result(guardrail, messages) + + assert not any(row.get("tool_call_id") == "hr_direct" for row in wire) + assert result["structured_messages"][3] == messages[3] + + +@pytest.mark.asyncio +async def test_retrieved_content_protected_when_mcp_tool_name_is_truncated(guardrail: HeadroomGuardrail): + """A long mcp____headroom_retrieve name is truncated past 64 chars in + the OpenAI-translated view the guardrail scans, dropping the suffix. The call + id read from the request's own Anthropic tool_use (never truncated) still + pairs the retrieved row so it is held back.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + truncate_tool_name, + ) + + long_name = "mcp__" + "s" * 45 + "__" + HEADROOM_RETRIEVE_TOOL_NAME + assert len(long_name) > 64 + truncated = truncate_tool_name(long_name) + assert not truncated.endswith(HEADROOM_RETRIEVE_TOOL_NAME) + + # What the guardrail scans: OpenAI-translated messages with the truncated name. + structured = [ + {"role": "system", "content": "sys " + "S" * 5000}, + {"role": "user", "content": "H" * 5000}, + { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "hr_long", "type": "function", "function": {"name": truncated, "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "hr_long", "content": "RETRIEVED BODY " + "R" * 5000}, + {"role": "assistant", "content": "Older. " + "O" * 5000}, + {"role": "user", "content": "summarize"}, + ] + # The request's own messages, untranslated: Anthropic tool_use carries the full name. + raw_messages = [ + {"role": "assistant", "content": [{"type": "tool_use", "id": "hr_long", "name": long_name, "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "hr_long", "content": "RETRIEVED BODY"}]}, + ] + + inputs = GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(structured))) + sent: dict = {} + + def _echo(**kwargs): + sent["messages"] = kwargs["json"]["messages"] + return _make_compress_response(json.loads(json.dumps(kwargs["json"]["messages"]))) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_echo): + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "claude-sonnet-4-5-20250929", "messages": raw_messages}, + input_type="request", + ) + + assert not any(row.get("tool_call_id") == "hr_long" for row in sent["messages"]) + assert result["structured_messages"][3] == structured[3] + assert any(row.get("content") == "H" * 5000 for row in sent["messages"]) + + +def test_raw_retrieve_call_ids_covers_both_shapes_and_ignores_others(): + """Retrieve ids are read from OpenAI tool_calls and Anthropic tool_use blocks; + non-retrieve calls, non-tool_use blocks, string content, and non-list inputs + yield nothing.""" + from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import _raw_retrieve_call_ids + + messages = [ + { + "role": "assistant", + "tool_calls": [ + {"id": "oa1", "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME}}, + {"id": "other", "function": {"name": "get_weather"}}, + {"id": "malformed", "function": {"name": 123}}, + {"id": "nofunc"}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "an1", "name": "mcp__hr__headroom_retrieve", "input": {}}, + {"type": "tool_use", "id": "an2", "name": "jira_get_issue", "input": {}}, + {"type": "text", "text": "noise"}, + ], + }, + {"role": "user", "content": "plain string content, not a list"}, + ] + + assert _raw_retrieve_call_ids(messages) == frozenset({"oa1", "an1"}) + assert _raw_retrieve_call_ids("not a list") == frozenset() + assert _raw_retrieve_call_ids(None) == frozenset() + + @pytest.mark.asyncio async def test_nothing_compressible_returns_inputs_untouched(guardrail: HeadroomGuardrail): """A single-turn request is all protected, so there is nothing to send and From 25987cb961567541922cef80e1433137f42c7b77 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 14:42:48 -0700 Subject: [PATCH 41/67] test(build): validate release wheel contracts --- .github/scripts/smoke_test_native_wheel.py | 68 ++++++++ .github/scripts/verify_linux_native_wheel.py | 161 +++++++++++++++++++ .github/workflows/test-rust.yml | 111 +++++++++++++ litellm-rust/crates/python-bridge/Cargo.toml | 1 + litellm-rust/crates/python-bridge/src/lib.rs | 8 + 5 files changed, 349 insertions(+) create mode 100644 .github/scripts/smoke_test_native_wheel.py create mode 100644 .github/scripts/verify_linux_native_wheel.py diff --git a/.github/scripts/smoke_test_native_wheel.py b/.github/scripts/smoke_test_native_wheel.py new file mode 100644 index 00000000000..577bb32fcf0 --- /dev/null +++ b/.github/scripts/smoke_test_native_wheel.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path +from typing import Final + +CHILD_SCRIPT: Final = """ +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +import sys + +native_path = Path(sys.argv[1]) +spec = spec_from_file_location("litellm.rust_bridge._native", native_path) +if spec is None or spec.loader is None: + raise RuntimeError("cannot create native extension import specification") +module = module_from_spec(spec) +spec.loader.exec_module(module) + +before = module.gil_stats() +if not isinstance(before.get("releases"), int): + raise AssertionError(f"unexpected gil_stats result: {before!r}") + +try: + module._panic_for_test() +except BaseException as error: + if type(error).__name__ != "PanicException": + raise AssertionError(f"expected PanicException, got {type(error).__name__}") from error +else: + raise AssertionError("Rust panic returned without raising") + +after = module.gil_stats() +if not isinstance(after.get("releases"), int): + raise AssertionError(f"native module unusable after panic: {after!r}") +""" + + +def main() -> int: + if len(sys.argv) != 2: + sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") + return 2 + + wheel: Final = Path(sys.argv[1]) + with tempfile.TemporaryDirectory() as temporary_directory, zipfile.ZipFile(wheel) as archive: + native_members: Final = tuple( + member + for member in archive.infolist() + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") + return 1 + + native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name + native_path.write_bytes(archive.read(native_members[0])) + result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False) + + if result.returncode != 0: + sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n") + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py new file mode 100644 index 00000000000..1de98ae0168 --- /dev/null +++ b/.github/scripts/verify_linux_native_wheel.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import importlib.util +import os +import re +import subprocess +import sys +import zipfile +from pathlib import Path, PurePosixPath +from typing import Final + + +def _loads_native_module(native_path: Path) -> bool: + module_spec: Final = importlib.util.spec_from_file_location("litellm.rust_bridge._native", native_path) + if module_spec is None or module_spec.loader is None: + return False + try: + native_module: Final = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(native_module) + except Exception as error: + sys.stderr.write(f"native module load failed: {error}\n") + return False + return True + + +def main() -> int: + if len(sys.argv) != 2: + sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") + return 2 + + wheel: Final = Path(sys.argv[1]) + with zipfile.ZipFile(wheel) as archive: + wheel_members: Final = archive.infolist() + native_members: Final = tuple( + member + for member in wheel_members + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") + return 1 + + unexpected_members: Final = tuple( + member.filename + for member in wheel_members + if member.filename.endswith((".pdb", ".dwp", ".rlib", ".rmeta", "Cargo.toml", "Cargo.lock")) + or any(part.endswith(".dSYM") for part in PurePosixPath(member.filename).parts) + ) + native_member: Final = native_members[0] + uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members) + native_path: Final = wheel.parent / "native" / Path(native_member.filename).name + native_path.parent.mkdir(parents=True, exist_ok=True) + native_path.write_bytes(archive.read(native_member)) + + wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) + if len(wheel_tags) != 4: + sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") + return 1 + + python_tag: Final = wheel_tags[1] + abi_tag: Final = wheel_tags[2] + platform_tag: Final = wheel_tags[3] + commit_sha: Final = os.environ.get("RELEASE_WHEEL_COMMIT_SHA", os.environ.get("GITHUB_SHA", "unknown")) + rustc_version: Final = subprocess.run( + ("rustc", "--version"), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text() + maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject) + if maturin_match is None: + sys.stderr.write("build-system does not pin an exact Maturin version\n") + return 1 + + maturin_version: Final = maturin_match.group(1) + native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100 + size_report: Final = "\n".join( + ( + "## Native wheel build report", + "", + "| Build | Value |", + "| --- | --- |", + f"| Commit | `{commit_sha}` |", + f"| Platform | `{platform_tag}` |", + f"| Python ABI | `{python_tag}-{abi_tag}` |", + f"| Rust compiler | `{rustc_version}` |", + f"| Maturin | `{maturin_version}` |", + "| Cargo profile | `release` |", + "", + "| Artifact | Size |", + "| --- | ---: |", + f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |", + f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |", + f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |", + f"| Native share | {native_percentage:.2f}% |", + "", + ) + ) + summary_path: Final = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path is None: + sys.stdout.write(size_report) + else: + Path(summary_path).write_text(size_report) + + sections: Final = subprocess.run( + ("readelf", "--sections", "--wide", native_path), + check=True, + capture_output=True, + text=True, + ).stdout + debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections) + debug_sections_absent: Final = not debug_sections + static_symbol_table_absent: Final = ".symtab" not in sections + + dynamic_symbols: Final = subprocess.run( + ("readelf", "--dyn-syms", "--wide", native_path), + check=True, + capture_output=True, + text=True, + ).stdout + extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols + native_module_loads: Final = _loads_native_module(native_path) + native_size_limit: Final = 20_000_000 + native_size_within_limit: Final = native_member.file_size <= native_size_limit + validations: Final = ( + ("Debug sections are absent", debug_sections_absent), + ("Static symbol table is absent", static_symbol_table_absent), + ("Python extension entry point is present", extension_entry_point_present), + ("Native module loads", native_module_loads), + ("Native extension does not exceed 20 MB", native_size_within_limit), + ("Wheel contents are valid", not unexpected_members), + ) + + verified_report: Final = size_report + "\n".join( + ("", "| Validation | Expected | Result |", "| --- | --- | :---: |") + + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations) + + ("",) + ) + report_path: Final = os.environ.get("RELEASE_WHEEL_REPORT") + if report_path is not None: + Path(report_path).write_text(size_report) + if summary_path is not None: + Path(summary_path).write_text(verified_report) + + if debug_sections: + sys.stderr.write(f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}\n") + if not static_symbol_table_absent: + sys.stderr.write(f"{native_member.filename} contains a static symbol table\n") + if not extension_entry_point_present: + sys.stderr.write("native extension does not export PyInit__native\n") + if not native_size_within_limit: + sys.stderr.write(f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB\n") + if unexpected_members: + sys.stderr.write(f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}\n") + + return 0 if all(passed for _, passed in validations) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 21e1bcb90c6..b05ddb81740 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -4,6 +4,10 @@ on: push: paths: - "litellm-rust/**" + - ".cargo/**" + - "pyproject.toml" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -13,6 +17,10 @@ on: - "litellm_**" paths: - "litellm-rust/**" + - ".cargo/**" + - "pyproject.toml" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" permissions: @@ -69,3 +77,106 @@ jobs: - name: Run core tests with Bedrock auth run: cargo test -p litellm-core --features bedrock-auth --locked + + release-wheel: + name: release wheel + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + pull-requests: write + env: + CARGO_TERM_COLOR: always + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Set up Rust + run: | + rustup toolchain install stable --profile minimal + rustup default stable + + - name: Cache release build + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + litellm-rust/target + key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-release- + + - name: Build release wheel + run: uv build --wheel --out-dir dist + + - name: Verify stripped native extension + env: + RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + RELEASE_WHEEL_REPORT: dist/release-wheel-report.md + run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl + + - name: Build panic contract wheel + run: >- + uv build --wheel --out-dir panic-dist + --config-setting "maturin.build-args=--features panic-test" + + - name: Smoke-test native panic unwinding + run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl + + - name: Report release wheel size on PR + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + COMMENT_MARKER: "" + REPORT_PATH: dist/release-wheel-report.md + with: + script: | + const fs = require("fs"); + const marker = process.env.COMMENT_MARKER; + const report = fs.readFileSync(process.env.REPORT_PATH, "utf8"); + const body = `${marker}\n${report}`; + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const existing = comments.find((comment) => comment.body?.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: Upload release wheel + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: litellm-release-wheel-linux-x86_64 + path: dist/*.whl + if-no-files-found: error diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index d461a483ae0..b1fdfd7677a 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -13,6 +13,7 @@ crate-type = ["cdylib"] default = ["abi3"] abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] +panic-test = [] [dependencies] litellm-core = { workspace = true, features = ["bedrock-auth"] } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f9e75f45f75..18e0b05cbb5 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -614,6 +614,12 @@ fn gil_stats(py: Python<'_>) -> PyResult> { Ok(stats.into_any().unbind()) } +#[cfg(feature = "panic-test")] +#[pyfunction] +fn _panic_for_test() { + panic!("intentional PyO3 panic smoke test"); +} + #[pymodule] fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { let py = module.py(); @@ -630,5 +636,7 @@ fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(achat_completions, module)?)?; module.add_class::()?; module.add_function(wrap_pyfunction!(gil_stats, module)?)?; + #[cfg(feature = "panic-test")] + module.add_function(wrap_pyfunction!(_panic_for_test, module)?)?; Ok(()) } From 2f362cfec2d122d4d2c14b73a988a20fe61e6629 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 14:59:03 -0700 Subject: [PATCH 42/67] fix(ci): preserve release wheel contract parity --- .github/scripts/verify_linux_native_wheel.py | 2 +- .github/workflows/test-rust.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index 1de98ae0168..b0ab34df98d 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -139,7 +139,7 @@ def main() -> int: ) report_path: Final = os.environ.get("RELEASE_WHEEL_REPORT") if report_path is not None: - Path(report_path).write_text(size_report) + Path(report_path).write_text(verified_report) if summary_path is not None: Path(summary_path).write_text(verified_report) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index b05ddb81740..1a9f36f24d2 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -132,7 +132,7 @@ jobs: - name: Build panic contract wheel run: >- uv build --wheel --out-dir panic-dist - --config-setting "maturin.build-args=--features panic-test" + --config-setting "maturin.build-args=--features panic-test,extension-module" - name: Smoke-test native panic unwinding run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl From 9dc9cd325cf5adca79d910a30c420381a28d1923 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 20:21:37 -0700 Subject: [PATCH 43/67] fix(ci): isolate release wheel reporting permissions --- .github/scripts/verify_linux_native_wheel.py | 3 - .../workflows/report-rust-release-wheel.yml | 95 +++++++++++++++++++ .github/workflows/test-rust.yml | 39 -------- 3 files changed, 95 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/report-rust-release-wheel.yml diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index b0ab34df98d..96264beb632 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -137,9 +137,6 @@ def main() -> int: + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations) + ("",) ) - report_path: Final = os.environ.get("RELEASE_WHEEL_REPORT") - if report_path is not None: - Path(report_path).write_text(verified_report) if summary_path is not None: Path(summary_path).write_text(verified_report) diff --git a/.github/workflows/report-rust-release-wheel.yml b/.github/workflows/report-rust-release-wheel.yml new file mode 100644 index 00000000000..76aefd70ef6 --- /dev/null +++ b/.github/workflows/report-rust-release-wheel.yml @@ -0,0 +1,95 @@ +name: Report LiteLLM Rust release wheel + +on: # zizmor: ignore[dangerous-triggers] reporter executes no PR code and consumes no PR artifacts or outputs + workflow_run: + workflows: + - LiteLLM Rust + types: + - completed + +permissions: {} + +jobs: + report-release-wheel: + name: report release wheel + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.path == '.github/workflows/test-rust.yml' && + github.event.workflow_run.head_repository.full_name == github.repository && + github.event.workflow_run.pull_requests[0].number != null + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write + + steps: + - name: Link release wheel report on PR + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + COMMENT_MARKER: "" + with: + script: | + const marker = process.env.COMMENT_MARKER; + const workflowRun = context.payload.workflow_run; + if ( + workflowRun.conclusion !== "success" || + workflowRun.event !== "pull_request" || + workflowRun.path !== ".github/workflows/test-rust.yml" || + workflowRun.head_repository?.full_name !== + `${context.repo.owner}/${context.repo.repo}` || + workflowRun.pull_requests?.length !== 1 + ) { + throw new Error("unexpected source workflow"); + } + const pullRequest = workflowRun.pull_requests[0]; + const pullRequestNumber = pullRequest.number; + const headSha = workflowRun.head_sha; + const runId = workflowRun.id; + if ( + !Number.isSafeInteger(pullRequestNumber) || + pullRequestNumber <= 0 || + !Number.isSafeInteger(runId) || + runId <= 0 || + !/^[0-9a-f]{40}$/.test(headSha) || + pullRequest.head?.sha !== headSha + ) { + throw new Error("invalid source workflow metadata"); + } + const runUrl = + `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + + `/actions/runs/${runId}`; + const body = [ + marker, + "## LiteLLM Rust workflow", + "", + `Workflow completed successfully for \`${headSha}\``, + "", + `[View workflow run](${runUrl})`, + ].join("\n"); + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + per_page: 100, + }); + const existing = comments.find( + (comment) => + comment.user?.login === "github-actions[bot]" && + comment.body?.startsWith(marker), + ); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body, + }); + } diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 1a9f36f24d2..b01e7fabe4a 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -84,7 +84,6 @@ jobs: timeout-minutes: 20 permissions: contents: read - pull-requests: write env: CARGO_TERM_COLOR: always @@ -126,7 +125,6 @@ jobs: - name: Verify stripped native extension env: RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - RELEASE_WHEEL_REPORT: dist/release-wheel-report.md run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl - name: Build panic contract wheel @@ -137,43 +135,6 @@ jobs: - name: Smoke-test native panic unwinding run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl - - name: Report release wheel size on PR - if: >- - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 - env: - COMMENT_MARKER: "" - REPORT_PATH: dist/release-wheel-report.md - with: - script: | - const fs = require("fs"); - const marker = process.env.COMMENT_MARKER; - const report = fs.readFileSync(process.env.REPORT_PATH, "utf8"); - const body = `${marker}\n${report}`; - const comments = await github.paginate(github.rest.issues.listComments, { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - per_page: 100, - }); - const existing = comments.find((comment) => comment.body?.includes(marker)); - if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body, - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body, - }); - } - - name: Upload release wheel uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: From e6a317e0790c575d6452f59c90e861f110b54a0c Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 20:53:17 -0700 Subject: [PATCH 44/67] fix(ci): enforce release wheel metadata contract --- .github/scripts/verify_linux_native_wheel.py | 110 ++++++++++-- .../test_verify_linux_native_wheel.py | 170 ++++++++++++++++++ 2 files changed, 266 insertions(+), 14 deletions(-) create mode 100644 tests/test_litellm/test_verify_linux_native_wheel.py diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index 96264beb632..61f176ab183 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -6,21 +6,44 @@ import re import subprocess import sys import zipfile +from email import policy +from email.parser import BytesParser +from itertools import product from pathlib import Path, PurePosixPath -from typing import Final +from types import ModuleType +from typing import Final, cast + +EXPECTED_PYTHON_TAG: Final = "cp310" +EXPECTED_ABI_TAG: Final = "abi3" +EXPECTED_PLATFORM_TAG: Final = "linux_x86_64" -def _loads_native_module(native_path: Path) -> bool: +def _dist_info_directory(member: zipfile.ZipInfo) -> str | None: + parts: Final = PurePosixPath(member.filename).parts + if not parts or not parts[0].endswith(".dist-info"): + return None + return parts[0] + + +def _wheel_metadata_tags(archive: zipfile.ZipFile, members: tuple[zipfile.ZipInfo, ...]) -> tuple[str, ...]: + if len(members) != 1: + return () + metadata: Final = BytesParser(policy=policy.default).parsebytes(archive.read(members[0])) + tags: Final = cast(list[str], metadata.get_all("Tag", [])) + return tuple(tag.strip() for tag in tags) + + +def _load_native_module(native_path: Path) -> ModuleType | None: module_spec: Final = importlib.util.spec_from_file_location("litellm.rust_bridge._native", native_path) if module_spec is None or module_spec.loader is None: - return False + return None try: native_module: Final = importlib.util.module_from_spec(module_spec) module_spec.loader.exec_module(native_module) except Exception as error: sys.stderr.write(f"native module load failed: {error}\n") - return False - return True + return None + return native_module def main() -> int: @@ -29,8 +52,39 @@ def main() -> int: return 2 wheel: Final = Path(sys.argv[1]) + wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) + if len(wheel_tags) != 4: + sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") + return 1 + + wheel_identity: Final = wheel_tags[0].split("-") + if len(wheel_identity) != 2 or wheel_identity[0] != "litellm" or not wheel_identity[1]: + sys.stderr.write(f"unexpected wheel identity: {wheel_tags[0]}\n") + return 1 + + expected_dist_info_directory: Final = f"{wheel_tags[0]}.dist-info" + expected_dist_info_directories: Final = frozenset((expected_dist_info_directory,)) + python_tag: Final = wheel_tags[1] + abi_tag: Final = wheel_tags[2] + platform_tag: Final = wheel_tags[3] + expanded_filename_tags: Final = frozenset( + "-".join(tag) for tag in product(python_tag.split("."), abi_tag.split("."), platform_tag.split(".")) + ) + with zipfile.ZipFile(wheel) as archive: wheel_members: Final = archive.infolist() + dist_info_directories: Final = frozenset( + directory for member in wheel_members if (directory := _dist_info_directory(member)) is not None + ) + required_dist_info_files: Final = ("METADATA", "RECORD", "WHEEL") + dist_info_file_counts: Final = { + filename: sum(member.filename == f"{expected_dist_info_directory}/{filename}" for member in wheel_members) + for filename in required_dist_info_files + } + wheel_metadata_members: Final = tuple( + member for member in wheel_members if member.filename == f"{expected_dist_info_directory}/WHEEL" + ) + wheel_metadata_tags: Final = _wheel_metadata_tags(archive, wheel_metadata_members) native_members: Final = tuple( member for member in wheel_members @@ -52,14 +106,10 @@ def main() -> int: native_path.parent.mkdir(parents=True, exist_ok=True) native_path.write_bytes(archive.read(native_member)) - wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) - if len(wheel_tags) != 4: - sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") - return 1 - - python_tag: Final = wheel_tags[1] - abi_tag: Final = wheel_tags[2] - platform_tag: Final = wheel_tags[3] + wheel_metadata_tags_match: Final = ( + len(wheel_metadata_tags) == len(expanded_filename_tags) + and frozenset(wheel_metadata_tags) == expanded_filename_tags + ) commit_sha: Final = os.environ.get("RELEASE_WHEEL_COMMIT_SHA", os.environ.get("GITHUB_SHA", "unknown")) rustc_version: Final = subprocess.run( ("rustc", "--version"), @@ -120,14 +170,26 @@ def main() -> int: text=True, ).stdout extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols - native_module_loads: Final = _loads_native_module(native_path) + native_module: Final = _load_native_module(native_path) + native_module_loads: Final = native_module is not None + panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") native_size_limit: Final = 20_000_000 native_size_within_limit: Final = native_member.file_size <= native_size_limit validations: Final = ( + (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), + (f"ABI tag is {EXPECTED_ABI_TAG}", abi_tag == EXPECTED_ABI_TAG), + (f"Platform tag is {EXPECTED_PLATFORM_TAG}", platform_tag == EXPECTED_PLATFORM_TAG), + ("Wheel dist-info directory matches the filename", dist_info_directories == expected_dist_info_directories), + ( + "Required dist-info files are present exactly once", + all(count == 1 for count in dist_info_file_counts.values()), + ), + ("Wheel metadata tags match the filename", wheel_metadata_tags_match), ("Debug sections are absent", debug_sections_absent), ("Static symbol table is absent", static_symbol_table_absent), ("Python extension entry point is present", extension_entry_point_present), ("Native module loads", native_module_loads), + ("Production module omits the panic test hook", panic_test_hook_absent), ("Native extension does not exceed 20 MB", native_size_within_limit), ("Wheel contents are valid", not unexpected_members), ) @@ -146,6 +208,26 @@ def main() -> int: sys.stderr.write(f"{native_member.filename} contains a static symbol table\n") if not extension_entry_point_present: sys.stderr.write("native extension does not export PyInit__native\n") + if python_tag != EXPECTED_PYTHON_TAG: + sys.stderr.write(f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}\n") + if abi_tag != EXPECTED_ABI_TAG: + sys.stderr.write(f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}\n") + if platform_tag != EXPECTED_PLATFORM_TAG: + sys.stderr.write(f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}\n") + if dist_info_directories != expected_dist_info_directories: + sys.stderr.write( + f"unexpected dist-info directories: expected {[expected_dist_info_directory]}, " + f"found {sorted(dist_info_directories)}\n" + ) + if any(count != 1 for count in dist_info_file_counts.values()): + sys.stderr.write(f"required dist-info file counts are invalid: {dist_info_file_counts}\n") + elif not wheel_metadata_tags_match: + sys.stderr.write( + f"WHEEL tags do not match filename: expected {sorted(expanded_filename_tags)}, " + f"found {sorted(wheel_metadata_tags)}\n" + ) + if native_module is not None and not panic_test_hook_absent: + sys.stderr.write("production native module exposes _panic_for_test\n") if not native_size_within_limit: sys.stderr.write(f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB\n") if unexpected_members: diff --git a/tests/test_litellm/test_verify_linux_native_wheel.py b/tests/test_litellm/test_verify_linux_native_wheel.py new file mode 100644 index 00000000000..86f5debfe3b --- /dev/null +++ b/tests/test_litellm/test_verify_linux_native_wheel.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import importlib.util +import subprocess +import sys +import zipfile +from collections.abc import Callable +from pathlib import Path +from types import ModuleType +from typing import Final, Protocol, cast + +import pytest + +_REPO_ROOT: Final = Path(__file__).resolve().parents[2] +_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "verify_linux_native_wheel.py" + + +class _VerifierModule(Protocol): + subprocess: ModuleType + _load_native_module: Callable[[Path], ModuleType | None] + main: Callable[[], int] + + +_SPEC: Final = importlib.util.spec_from_file_location("verify_linux_native_wheel", _MODULE_PATH) +assert _SPEC is not None and _SPEC.loader is not None +_LOADED_VERIFIER: Final = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _LOADED_VERIFIER +_SPEC.loader.exec_module(_LOADED_VERIFIER) +verifier: Final = cast(_VerifierModule, _LOADED_VERIFIER) + +_EXPECTED_TAG: Final = "cp310-abi3-linux_x86_64" +_NATIVE_MEMBER: Final = "litellm/rust_bridge/_native.abi3.so" +_DIST_INFO: Final = "litellm-1.100.0.dist-info" + + +def _write_wheel( + tmp_path: Path, + *, + filename_tag: str, + metadata_tags: tuple[str, ...] | None = (_EXPECTED_TAG,), + dist_info: str = _DIST_INFO, + duplicate_wheel: bool = False, +) -> Path: + wheel: Final = tmp_path / f"litellm-1.100.0-{filename_tag}.whl" + with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(_NATIVE_MEMBER, b"synthetic native extension") + archive.writestr( + f"{dist_info}/METADATA", + "Metadata-Version: 2.1\nName: litellm\nVersion: 1.100.0\n", + ) + archive.writestr( + f"{dist_info}/RECORD", + f"{_NATIVE_MEMBER},,\n{dist_info}/WHEEL,,\n", + ) + if metadata_tags is not None: + wheel_metadata: Final = ( + "Wheel-Version: 1.0\nGenerator: regression-test\nRoot-Is-Purelib: false\n" + + "".join(f"Tag: {tag}\n" for tag in metadata_tags) + ) + archive.writestr(f"{dist_info}/WHEEL", wheel_metadata) + if duplicate_wheel: + archive.writestr(f"{dist_info}/WHEEL", wheel_metadata) + return wheel + + +def _fake_subprocess_run(command: tuple[str, ...], **_: object) -> subprocess.CompletedProcess[str]: + if command == ("rustc", "--version"): + stdout = "rustc 1.98.0 (regression-test)\n" + elif "--sections" in command: + stdout = "[ 1] .text PROGBITS\n" + elif "--dyn-syms" in command: + stdout = "PyInit__native\n" + else: + raise AssertionError(f"unexpected subprocess command: {command}") + return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr="") + + +def _run_verifier( + monkeypatch: pytest.MonkeyPatch, + wheel: Path, + *, + exposes_panic: bool = False, +) -> int: + native_module: Final = ModuleType("litellm.rust_bridge._native") + if exposes_panic: + setattr(native_module, "_panic_for_test", lambda: None) + + def _fake_load_native_module(_: Path) -> ModuleType: + return native_module + + monkeypatch.setattr(verifier, "_load_native_module", _fake_load_native_module) + monkeypatch.setattr(verifier.subprocess, "run", _fake_subprocess_run) + monkeypatch.setattr(sys, "argv", [str(_MODULE_PATH), str(wheel)]) + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(wheel.parent / "summary.md")) + return verifier.main() + + +def test_accepts_expected_release_wheel_tags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + + assert _run_verifier(monkeypatch, wheel) == 0 + + +def test_rejects_cp312_version_specific_wheel(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + tag: Final = "cp312-cp312-linux_x86_64" + wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +def test_rejects_non_linux_platform_tag(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + tag: Final = "cp310-abi3-win_amd64" + wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +@pytest.mark.parametrize( + "metadata_tags", + [None, ("cp312-cp312-linux_x86_64",)], + ids=["missing", "mismatched"], +) +def test_rejects_missing_or_mismatched_wheel_metadata_tag( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + metadata_tags: tuple[str, ...] | None, +) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=metadata_tags) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +def test_rejects_wheel_metadata_from_wrong_dist_info_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + dist_info="decoy-1.0.0.dist-info", + ) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +def test_rejects_duplicate_wheel_metadata_tags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + metadata_tags=(_EXPECTED_TAG, _EXPECTED_TAG), + ) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.warns(UserWarning, match="Duplicate name"): + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + duplicate_wheel=True, + ) + + assert _run_verifier(monkeypatch, wheel) == 1 + + +def test_rejects_production_module_exposing_panic_hook(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + + assert _run_verifier(monkeypatch, wheel, exposes_panic=True) == 1 From ef9a207ed545e740863926647af148e5195285c9 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 05:44:35 -0700 Subject: [PATCH 45/67] fix(ci): harden release wheel reporting --- .../workflows/report-rust-release-wheel.yml | 43 +++++++++++++++++-- .github/workflows/test-rust.yml | 26 ++--------- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/.github/workflows/report-rust-release-wheel.yml b/.github/workflows/report-rust-release-wheel.yml index 76aefd70ef6..1d93b56f77f 100644 --- a/.github/workflows/report-rust-release-wheel.yml +++ b/.github/workflows/report-rust-release-wheel.yml @@ -9,11 +9,14 @@ on: # zizmor: ignore[dangerous-triggers] reporter executes no PR code and consum permissions: {} +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || github.event.workflow_run.id }} + cancel-in-progress: false + jobs: report-release-wheel: name: report release wheel if: >- - github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.path == '.github/workflows/test-rust.yml' && github.event.workflow_run.head_repository.full_name == github.repository && @@ -21,7 +24,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 permissions: - issues: write + issues: write # PR comments use the issues API + pull-requests: read # Current-head validation rejects stale workflow runs steps: - name: Link release wheel report on PR @@ -32,8 +36,19 @@ jobs: script: | const marker = process.env.COMMENT_MARKER; const workflowRun = context.payload.workflow_run; + const allowedConclusions = new Set([ + "action_required", + "cancelled", + "failure", + "neutral", + "skipped", + "stale", + "startup_failure", + "success", + "timed_out", + ]); if ( - workflowRun.conclusion !== "success" || + !allowedConclusions.has(workflowRun.conclusion) || workflowRun.event !== "pull_request" || workflowRun.path !== ".github/workflows/test-rust.yml" || workflowRun.head_repository?.full_name !== @@ -59,11 +74,15 @@ jobs: const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + `/actions/runs/${runId}`; + const result = + workflowRun.conclusion === "success" + ? "successfully" + : `with \`${workflowRun.conclusion}\``; const body = [ marker, "## LiteLLM Rust workflow", "", - `Workflow completed successfully for \`${headSha}\``, + `Workflow completed ${result} for \`${headSha}\``, "", `[View workflow run](${runUrl})`, ].join("\n"); @@ -78,6 +97,22 @@ jobs: comment.user?.login === "github-actions[bot]" && comment.body?.startsWith(marker), ); + const currentPullRequest = ( + await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullRequestNumber, + }) + ).data; + if ( + currentPullRequest.state !== "open" || + currentPullRequest.head.repo?.full_name !== + `${context.repo.owner}/${context.repo.repo}` || + currentPullRequest.head.sha !== headSha + ) { + core.info("source workflow no longer matches the current pull request head"); + return; + } if (existing) { await github.rest.issues.updateComment({ owner: context.repo.owner, diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index b01e7fabe4a..77da358a2fc 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -108,25 +108,9 @@ jobs: rustup toolchain install stable --profile minimal rustup default stable - - name: Cache release build - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - litellm-rust/target - key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-release- - - name: Build release wheel run: uv build --wheel --out-dir dist - - name: Verify stripped native extension - env: - RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl - - name: Build panic contract wheel run: >- uv build --wheel --out-dir panic-dist @@ -135,9 +119,7 @@ jobs: - name: Smoke-test native panic unwinding run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl - - name: Upload release wheel - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: litellm-release-wheel-linux-x86_64 - path: dist/*.whl - if-no-files-found: error + - name: Verify stripped native extension + env: + RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl From 38150dfc2c50526664125c25c6eabf07d06e07ed Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 06:37:39 -0700 Subject: [PATCH 46/67] fix(ci): pin workflow toolchain dependencies --- .circleci/config.yml | 8 ++--- .../actions/setup-uv-with-retries/action.yml | 14 +++----- .github/workflows/test-rust.yml | 12 +++---- rust-toolchain.toml | 4 +++ .../test_circleci_rust_toolchain.py | 33 ++++++++++++++----- 5 files changed, 42 insertions(+), 29 deletions(-) create mode 100644 rust-toolchain.toml diff --git a/.circleci/config.yml b/.circleci/config.yml index 55fa9410845..dfc539fb80e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,10 +112,10 @@ commands: node --version npm --version install_rust: - description: "Install pinned rustup (1.28.2) and Rust toolchain (1.97.1) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself." + description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself." steps: - run: - name: Install Rust (rustup 1.28.2, toolchain 1.97.1) + name: Install Rust (rustup 1.28.2, toolchain 1.98.0) command: | case "$(uname -m)" in x86_64) @@ -135,7 +135,7 @@ commands: "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init" echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - chmod +x /tmp/rustup-init - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.97.1 + /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0 rm -f /tmp/rustup-init echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.cargo/bin:$PATH" @@ -300,7 +300,7 @@ jobs: if ($rustupActual -ne $rustupExpected) { throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual" } - & $rustupInit -y --profile minimal --default-toolchain stable + & $rustupInit -y --profile minimal --default-toolchain 1.98.0 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/.github/actions/setup-uv-with-retries/action.yml b/.github/actions/setup-uv-with-retries/action.yml index 1627038dc3d..98ff91f0283 100644 --- a/.github/actions/setup-uv-with-retries/action.yml +++ b/.github/actions/setup-uv-with-retries/action.yml @@ -1,11 +1,7 @@ name: "Set up uv with retries" description: >- - Install uv via astral-sh/setup-uv, retrying on transient failures. Even with - an exact pinned version, the action resolves the artifact URL by fetching - https://raw.githubusercontent.com/astral-sh/versions/main/v1/uv.ndjson in a - single request with no retry, timeout, or fallback, so one connection-level - network error ("fetch failed") fails the whole job before any test runs. - Retrying the full step covers the manifest fetch and the binary download. + Install uv via astral-sh/setup-uv, retrying the full setup step so manifest + resolution and binary downloads get fresh attempts after transient failures. inputs: version: @@ -18,7 +14,7 @@ runs: - name: Set up uv (attempt 1) id: attempt-1 continue-on-error: true - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: version: ${{ inputs.version }} @@ -31,7 +27,7 @@ runs: id: attempt-2 if: steps.attempt-1.outcome == 'failure' continue-on-error: true - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: version: ${{ inputs.version }} @@ -42,6 +38,6 @@ runs: - name: Set up uv (attempt 3) if: steps.attempt-2.outcome == 'failure' - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: version: ${{ inputs.version }} diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 77da358a2fc..aada0fcf239 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -6,6 +6,7 @@ on: - "litellm-rust/**" - ".cargo/**" - "pyproject.toml" + - "rust-toolchain.toml" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" @@ -19,6 +20,7 @@ on: - "litellm-rust/**" - ".cargo/**" - "pyproject.toml" + - "rust-toolchain.toml" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" @@ -48,9 +50,7 @@ jobs: persist-credentials: false - name: Set up Rust - run: | - rustup toolchain install stable --profile minimal --component clippy,rustfmt - rustup default stable + run: rustup toolchain install - name: Cache Cargo registry and target uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 @@ -59,7 +59,7 @@ jobs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-cargo-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-cargo-${{ hashFiles('rust-toolchain.toml', 'litellm-rust/Cargo.lock') }} restore-keys: | ${{ runner.os }}-cargo- @@ -104,9 +104,7 @@ jobs: version: "0.10.9" - name: Set up Rust - run: | - rustup toolchain install stable --profile minimal - rustup default stable + run: rustup toolchain install - name: Build release wheel run: uv build --wheel --out-dir dist diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000000..a1598ccbb34 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.98.0" +profile = "minimal" +components = ["rustfmt", "clippy"] diff --git a/tests/test_litellm/test_circleci_rust_toolchain.py b/tests/test_litellm/test_circleci_rust_toolchain.py index c35ced51e16..800ca21b95d 100644 --- a/tests/test_litellm/test_circleci_rust_toolchain.py +++ b/tests/test_litellm/test_circleci_rust_toolchain.py @@ -17,28 +17,27 @@ Two invariants are pinned here: Windows job, so the check accepts either. A new job that syncs without one falls back to the unpinned path, which is exactly the regression a static check catches at PR time and a green CI run does not. - 2. `install_rust` itself pins what it downloads: an explicit rustup version in - the URL, a verified SHA-256, and an exact toolchain version rather than a - channel name. - -The Windows job predates `install_rust` and provisions its toolchain inline, so -invariant 2 is scoped to `install_rust`; invariant 1 covers both. + 2. Both installers pin what they download: an explicit rustup version, a + verified SHA-256, and the exact toolchain in `rust-toolchain.toml`. """ from __future__ import annotations import re from pathlib import Path +from typing import Final import pytest import yaml REPO_ROOT = Path(__file__).resolve().parents[2] CONFIG = REPO_ROOT / ".circleci" / "config.yml" +TOOLCHAIN: Final = REPO_ROOT / "rust-toolchain.toml" BUILDS_WORKSPACE = re.compile(r"\buv\s+(?:sync|build)\b") RUSTUP_ARCHIVE_URL = re.compile(r"https://static\.rust-lang\.org/rustup/archive/\d+\.\d+\.\d+/") -EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?\d+\.\d+\.\d+\"?") +EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?(\d+\.\d+\.\d+)\"?") +TOOLCHAIN_CHANNEL: Final = re.compile(r'^channel = "(\d+\.\d+\.\d+)"$', re.MULTILINE) def _config() -> dict[str, object]: @@ -57,6 +56,12 @@ def _step_text(step: object) -> str: return "" +def _pinned_toolchain() -> str: + match: Final = TOOLCHAIN_CHANNEL.search(TOOLCHAIN.read_text()) + assert match is not None, "rust-toolchain.toml must pin an exact channel" + return match.group(1) + + def _without_comments(text: str) -> str: return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) @@ -142,7 +147,17 @@ def test_install_rust_verifies_the_installer_checksum(install_rust_command: str) def test_install_rust_pins_an_exact_toolchain_version(install_rust_command: str) -> None: - assert EXACT_TOOLCHAIN.search(install_rust_command), ( - "install_rust must pin an exact toolchain version (e.g. 1.97.1); a channel name like " + match: Final = EXACT_TOOLCHAIN.search(install_rust_command) + assert match is not None, ( + "install_rust must pin an exact toolchain version (e.g. 1.98.0); a channel name like " "stable/beta/nightly makes the compiler drift with whatever upstream published that day" ) + assert match.group(1) == _pinned_toolchain() + + +def test_windows_installer_matches_the_repo_toolchain() -> None: + windows_steps: Final = _step_lists()["job using_litellm_on_windows"] + windows_command: Final = "\n".join(_step_text(step) for step in windows_steps) + match: Final = EXACT_TOOLCHAIN.search(windows_command) + assert match is not None + assert match.group(1) == _pinned_toolchain() From cbb8a1784db64b8433c07d09af2a114993b717ff Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 06:49:43 -0700 Subject: [PATCH 47/67] chore(ci): extract setup-uv pin --- .github/actions/setup-uv-with-retries/action.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-uv-with-retries/action.yml b/.github/actions/setup-uv-with-retries/action.yml index 98ff91f0283..1627038dc3d 100644 --- a/.github/actions/setup-uv-with-retries/action.yml +++ b/.github/actions/setup-uv-with-retries/action.yml @@ -1,7 +1,11 @@ name: "Set up uv with retries" description: >- - Install uv via astral-sh/setup-uv, retrying the full setup step so manifest - resolution and binary downloads get fresh attempts after transient failures. + Install uv via astral-sh/setup-uv, retrying on transient failures. Even with + an exact pinned version, the action resolves the artifact URL by fetching + https://raw.githubusercontent.com/astral-sh/versions/main/v1/uv.ndjson in a + single request with no retry, timeout, or fallback, so one connection-level + network error ("fetch failed") fails the whole job before any test runs. + Retrying the full step covers the manifest fetch and the binary download. inputs: version: @@ -14,7 +18,7 @@ runs: - name: Set up uv (attempt 1) id: attempt-1 continue-on-error: true - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 with: version: ${{ inputs.version }} @@ -27,7 +31,7 @@ runs: id: attempt-2 if: steps.attempt-1.outcome == 'failure' continue-on-error: true - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 with: version: ${{ inputs.version }} @@ -38,6 +42,6 @@ runs: - name: Set up uv (attempt 3) if: steps.attempt-2.outcome == 'failure' - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 with: version: ${{ inputs.version }} From ce0c85ea691922d1e2f06bec93df10b7dc7f43ad Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 12:06:00 -0700 Subject: [PATCH 48/67] refactor(rust): colocate native wheel contract checks --- .github/workflows/test-rust.yml | 12 ++-- .../rust_bridge}/smoke_test_native_wheel.py | 0 .../rust_bridge}/verify_linux_native_wheel.py | 57 ++++++++++++---- .../test_verify_linux_native_wheel.py | 65 +++++++------------ 4 files changed, 74 insertions(+), 60 deletions(-) rename {.github/scripts => litellm/rust_bridge}/smoke_test_native_wheel.py (100%) rename {.github/scripts => litellm/rust_bridge}/verify_linux_native_wheel.py (86%) rename tests/test_litellm/{ => rust_bridge}/test_verify_linux_native_wheel.py (62%) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index aada0fcf239..271c73733a3 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -7,8 +7,8 @@ on: - ".cargo/**" - "pyproject.toml" - "rust-toolchain.toml" - - ".github/scripts/smoke_test_native_wheel.py" - - ".github/scripts/verify_linux_native_wheel.py" + - "litellm/rust_bridge/smoke_test_native_wheel.py" + - "litellm/rust_bridge/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -21,8 +21,8 @@ on: - ".cargo/**" - "pyproject.toml" - "rust-toolchain.toml" - - ".github/scripts/smoke_test_native_wheel.py" - - ".github/scripts/verify_linux_native_wheel.py" + - "litellm/rust_bridge/smoke_test_native_wheel.py" + - "litellm/rust_bridge/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" permissions: @@ -115,9 +115,9 @@ jobs: --config-setting "maturin.build-args=--features panic-test,extension-module" - name: Smoke-test native panic unwinding - run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl + run: python litellm/rust_bridge/smoke_test_native_wheel.py panic-dist/*.whl - name: Verify stripped native extension env: RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl + run: python litellm/rust_bridge/verify_linux_native_wheel.py dist/*.whl diff --git a/.github/scripts/smoke_test_native_wheel.py b/litellm/rust_bridge/smoke_test_native_wheel.py similarity index 100% rename from .github/scripts/smoke_test_native_wheel.py rename to litellm/rust_bridge/smoke_test_native_wheel.py diff --git a/.github/scripts/verify_linux_native_wheel.py b/litellm/rust_bridge/verify_linux_native_wheel.py similarity index 86% rename from .github/scripts/verify_linux_native_wheel.py rename to litellm/rust_bridge/verify_linux_native_wheel.py index 61f176ab183..783de87a8b2 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/litellm/rust_bridge/verify_linux_native_wheel.py @@ -6,18 +6,40 @@ import re import subprocess import sys import zipfile +from collections.abc import Callable, Mapping, Sequence from email import policy from email.parser import BytesParser from itertools import product from pathlib import Path, PurePosixPath from types import ModuleType -from typing import Final, cast +from typing import Final, Protocol, cast EXPECTED_PYTHON_TAG: Final = "cp310" EXPECTED_ABI_TAG: Final = "abi3" EXPECTED_PLATFORM_TAG: Final = "linux_x86_64" +class CommandRunner(Protocol): + def __call__( + self, + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +def _run_command( + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, check=check, capture_output=capture_output, text=text) + + def _dist_info_directory(member: zipfile.ZipInfo) -> str | None: parts: Final = PurePosixPath(member.filename).parts if not parts or not parts[0].endswith(".dist-info"): @@ -46,12 +68,19 @@ def _load_native_module(native_path: Path) -> ModuleType | None: return native_module -def main() -> int: - if len(sys.argv) != 2: - sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") +def main( + argv: Sequence[str] | None = None, + environment: Mapping[str, str] | None = None, + load_native_module: Callable[[Path], ModuleType | None] = _load_native_module, + run_command: CommandRunner = _run_command, +) -> int: + arguments: Final = tuple(sys.argv if argv is None else argv) + resolved_environment: Final = os.environ if environment is None else environment + if len(arguments) != 2: + sys.stderr.write(f"usage: {Path(arguments[0]).name} WHEEL\n") return 2 - wheel: Final = Path(sys.argv[1]) + wheel: Final = Path(arguments[1]) wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) if len(wheel_tags) != 4: sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") @@ -110,8 +139,10 @@ def main() -> int: len(wheel_metadata_tags) == len(expanded_filename_tags) and frozenset(wheel_metadata_tags) == expanded_filename_tags ) - commit_sha: Final = os.environ.get("RELEASE_WHEEL_COMMIT_SHA", os.environ.get("GITHUB_SHA", "unknown")) - rustc_version: Final = subprocess.run( + commit_sha: Final = resolved_environment.get( + "RELEASE_WHEEL_COMMIT_SHA", resolved_environment.get("GITHUB_SHA", "unknown") + ) + rustc_version: Final = run_command( ("rustc", "--version"), check=True, capture_output=True, @@ -147,14 +178,14 @@ def main() -> int: "", ) ) - summary_path: Final = os.environ.get("GITHUB_STEP_SUMMARY") + summary_path: Final = resolved_environment.get("GITHUB_STEP_SUMMARY") if summary_path is None: sys.stdout.write(size_report) else: Path(summary_path).write_text(size_report) - sections: Final = subprocess.run( - ("readelf", "--sections", "--wide", native_path), + sections: Final = run_command( + ("readelf", "--sections", "--wide", str(native_path)), check=True, capture_output=True, text=True, @@ -163,14 +194,14 @@ def main() -> int: debug_sections_absent: Final = not debug_sections static_symbol_table_absent: Final = ".symtab" not in sections - dynamic_symbols: Final = subprocess.run( - ("readelf", "--dyn-syms", "--wide", native_path), + dynamic_symbols: Final = run_command( + ("readelf", "--dyn-syms", "--wide", str(native_path)), check=True, capture_output=True, text=True, ).stdout extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols - native_module: Final = _load_native_module(native_path) + native_module: Final = load_native_module(native_path) native_module_loads: Final = native_module is not None panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") native_size_limit: Final = 20_000_000 diff --git a/tests/test_litellm/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py similarity index 62% rename from tests/test_litellm/test_verify_linux_native_wheel.py rename to tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py index 86f5debfe3b..a4291ce0a65 100644 --- a/tests/test_litellm/test_verify_linux_native_wheel.py +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -1,32 +1,16 @@ from __future__ import annotations -import importlib.util import subprocess -import sys import zipfile -from collections.abc import Callable from pathlib import Path from types import ModuleType -from typing import Final, Protocol, cast +from typing import Final import pytest -_REPO_ROOT: Final = Path(__file__).resolve().parents[2] -_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "verify_linux_native_wheel.py" +from litellm.rust_bridge import verify_linux_native_wheel as verifier - -class _VerifierModule(Protocol): - subprocess: ModuleType - _load_native_module: Callable[[Path], ModuleType | None] - main: Callable[[], int] - - -_SPEC: Final = importlib.util.spec_from_file_location("verify_linux_native_wheel", _MODULE_PATH) -assert _SPEC is not None and _SPEC.loader is not None -_LOADED_VERIFIER: Final = importlib.util.module_from_spec(_SPEC) -sys.modules[_SPEC.name] = _LOADED_VERIFIER -_SPEC.loader.exec_module(_LOADED_VERIFIER) -verifier: Final = cast(_VerifierModule, _LOADED_VERIFIER) +_MODULE_PATH: Final = Path(verifier.__file__) _EXPECTED_TAG: Final = "cp310-abi3-linux_x86_64" _NATIVE_MEMBER: Final = "litellm/rust_bridge/_native.abi3.so" @@ -76,7 +60,6 @@ def _fake_subprocess_run(command: tuple[str, ...], **_: object) -> subprocess.Co def _run_verifier( - monkeypatch: pytest.MonkeyPatch, wheel: Path, *, exposes_panic: bool = False, @@ -88,31 +71,33 @@ def _run_verifier( def _fake_load_native_module(_: Path) -> ModuleType: return native_module - monkeypatch.setattr(verifier, "_load_native_module", _fake_load_native_module) - monkeypatch.setattr(verifier.subprocess, "run", _fake_subprocess_run) - monkeypatch.setattr(sys, "argv", [str(_MODULE_PATH), str(wheel)]) - monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(wheel.parent / "summary.md")) - return verifier.main() + environment: Final = {"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")} + return verifier.main( + (str(_MODULE_PATH), str(wheel)), + environment, + _fake_load_native_module, + _fake_subprocess_run, + ) -def test_accepts_expected_release_wheel_tags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_accepts_expected_release_wheel_tags(tmp_path: Path) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) - assert _run_verifier(monkeypatch, wheel) == 0 + assert _run_verifier(wheel) == 0 -def test_rejects_cp312_version_specific_wheel(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_cp312_version_specific_wheel(tmp_path: Path) -> None: tag: Final = "cp312-cp312-linux_x86_64" wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 -def test_rejects_non_linux_platform_tag(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None: tag: Final = "cp310-abi3-win_amd64" wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 @pytest.mark.parametrize( @@ -122,17 +107,15 @@ def test_rejects_non_linux_platform_tag(tmp_path: Path, monkeypatch: pytest.Monk ) def test_rejects_missing_or_mismatched_wheel_metadata_tag( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, metadata_tags: tuple[str, ...] | None, ) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=metadata_tags) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 def test_rejects_wheel_metadata_from_wrong_dist_info_directory( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: wheel: Final = _write_wheel( tmp_path, @@ -140,20 +123,20 @@ def test_rejects_wheel_metadata_from_wrong_dist_info_directory( dist_info="decoy-1.0.0.dist-info", ) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 -def test_rejects_duplicate_wheel_metadata_tags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_duplicate_wheel_metadata_tags(tmp_path: Path) -> None: wheel: Final = _write_wheel( tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=(_EXPECTED_TAG, _EXPECTED_TAG), ) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 -def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path) -> None: with pytest.warns(UserWarning, match="Duplicate name"): wheel: Final = _write_wheel( tmp_path, @@ -161,10 +144,10 @@ def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path, monkeypatch: pyte duplicate_wheel=True, ) - assert _run_verifier(monkeypatch, wheel) == 1 + assert _run_verifier(wheel) == 1 -def test_rejects_production_module_exposing_panic_hook(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_production_module_exposing_panic_hook(tmp_path: Path) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) - assert _run_verifier(monkeypatch, wheel, exposes_panic=True) == 1 + assert _run_verifier(wheel, exposes_panic=True) == 1 From 814204e21f87c65234d5167cff92c92d357a5587 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 16:06:39 -0700 Subject: [PATCH 49/67] fix(rust): satisfy native wheel verifier lint --- .../rust_bridge/verify_linux_native_wheel.py | 93 +++++++++++-------- .../test_verify_linux_native_wheel.py | 44 +++++---- 2 files changed, 80 insertions(+), 57 deletions(-) diff --git a/litellm/rust_bridge/verify_linux_native_wheel.py b/litellm/rust_bridge/verify_linux_native_wheel.py index 783de87a8b2..899e2a211c0 100644 --- a/litellm/rust_bridge/verify_linux_native_wheel.py +++ b/litellm/rust_bridge/verify_linux_native_wheel.py @@ -7,12 +7,10 @@ import subprocess import sys import zipfile from collections.abc import Callable, Mapping, Sequence -from email import policy -from email.parser import BytesParser from itertools import product from pathlib import Path, PurePosixPath -from types import ModuleType -from typing import Final, Protocol, cast +from types import MappingProxyType, ModuleType +from typing import Final, Protocol EXPECTED_PYTHON_TAG: Final = "cp310" EXPECTED_ABI_TAG: Final = "abi3" @@ -50,9 +48,8 @@ def _dist_info_directory(member: zipfile.ZipInfo) -> str | None: def _wheel_metadata_tags(archive: zipfile.ZipFile, members: tuple[zipfile.ZipInfo, ...]) -> tuple[str, ...]: if len(members) != 1: return () - metadata: Final = BytesParser(policy=policy.default).parsebytes(archive.read(members[0])) - tags: Final = cast(list[str], metadata.get_all("Tag", [])) - return tuple(tag.strip() for tag in tags) + lines: Final = archive.read(members[0]).splitlines() + return tuple(line.removeprefix(b"Tag:").strip().decode("ascii") for line in lines if line.startswith(b"Tag:")) def _load_native_module(native_path: Path) -> ModuleType | None: @@ -62,7 +59,7 @@ def _load_native_module(native_path: Path) -> ModuleType | None: try: native_module: Final = importlib.util.module_from_spec(module_spec) module_spec.loader.exec_module(native_module) - except Exception as error: + except Exception as error: # noqa: BLE001 # native module initialization can raise arbitrary exceptions sys.stderr.write(f"native module load failed: {error}\n") return None return native_module @@ -106,10 +103,14 @@ def main( directory for member in wheel_members if (directory := _dist_info_directory(member)) is not None ) required_dist_info_files: Final = ("METADATA", "RECORD", "WHEEL") - dist_info_file_counts: Final = { - filename: sum(member.filename == f"{expected_dist_info_directory}/{filename}" for member in wheel_members) - for filename in required_dist_info_files - } + dist_info_file_counts: Final = MappingProxyType( + { + filename: sum( + member.filename == f"{expected_dist_info_directory}/{filename}" for member in wheel_members + ) + for filename in required_dist_info_files + } + ) wheel_metadata_members: Final = tuple( member for member in wheel_members if member.filename == f"{expected_dist_info_directory}/WHEEL" ) @@ -233,36 +234,46 @@ def main( if summary_path is not None: Path(summary_path).write_text(verified_report) - if debug_sections: - sys.stderr.write(f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}\n") - if not static_symbol_table_absent: - sys.stderr.write(f"{native_member.filename} contains a static symbol table\n") - if not extension_entry_point_present: - sys.stderr.write("native extension does not export PyInit__native\n") - if python_tag != EXPECTED_PYTHON_TAG: - sys.stderr.write(f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}\n") - if abi_tag != EXPECTED_ABI_TAG: - sys.stderr.write(f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}\n") - if platform_tag != EXPECTED_PLATFORM_TAG: - sys.stderr.write(f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}\n") - if dist_info_directories != expected_dist_info_directories: - sys.stderr.write( - f"unexpected dist-info directories: expected {[expected_dist_info_directory]}, " - f"found {sorted(dist_info_directories)}\n" + invalid_dist_info_files: Final = any(count != 1 for count in dist_info_file_counts.values()) + validation_errors: Final = tuple( + message + for failed, message in ( + (bool(debug_sections), f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}"), + (not static_symbol_table_absent, f"{native_member.filename} contains a static symbol table"), + (not extension_entry_point_present, "native extension does not export PyInit__native"), + ( + python_tag != EXPECTED_PYTHON_TAG, + f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}", + ), + (abi_tag != EXPECTED_ABI_TAG, f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}"), + ( + platform_tag != EXPECTED_PLATFORM_TAG, + f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}", + ), + ( + dist_info_directories != expected_dist_info_directories, + f"unexpected dist-info directories: expected {expected_dist_info_directory}, " + f"found {', '.join(sorted(dist_info_directories))}", + ), + (invalid_dist_info_files, f"required dist-info file counts are invalid: {dist_info_file_counts}"), + ( + not invalid_dist_info_files and not wheel_metadata_tags_match, + f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, " + f"found {', '.join(sorted(wheel_metadata_tags))}", + ), + ( + native_module is not None and not panic_test_hook_absent, + "production native module exposes _panic_for_test", + ), + ( + not native_size_within_limit, + f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB", + ), + (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"), ) - if any(count != 1 for count in dist_info_file_counts.values()): - sys.stderr.write(f"required dist-info file counts are invalid: {dist_info_file_counts}\n") - elif not wheel_metadata_tags_match: - sys.stderr.write( - f"WHEEL tags do not match filename: expected {sorted(expanded_filename_tags)}, " - f"found {sorted(wheel_metadata_tags)}\n" - ) - if native_module is not None and not panic_test_hook_absent: - sys.stderr.write("production native module exposes _panic_for_test\n") - if not native_size_within_limit: - sys.stderr.write(f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB\n") - if unexpected_members: - sys.stderr.write(f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}\n") + if failed + ) + sys.stderr.write("".join(f"{message}\n" for message in validation_errors)) return 0 if all(passed for _, passed in validations) else 1 diff --git a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py index a4291ce0a65..8d0082dddc1 100644 --- a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -3,7 +3,7 @@ from __future__ import annotations import subprocess import zipfile from pathlib import Path -from types import ModuleType +from types import MappingProxyType, ModuleType from typing import Final import pytest @@ -47,16 +47,26 @@ def _write_wheel( return wheel -def _fake_subprocess_run(command: tuple[str, ...], **_: object) -> subprocess.CompletedProcess[str]: +def _fake_subprocess_run( + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + assert check and capture_output and text if command == ("rustc", "--version"): - stdout = "rustc 1.98.0 (regression-test)\n" - elif "--sections" in command: - stdout = "[ 1] .text PROGBITS\n" - elif "--dyn-syms" in command: - stdout = "PyInit__native\n" - else: - raise AssertionError(f"unexpected subprocess command: {command}") - return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr="") + return subprocess.CompletedProcess(command, 0, stdout="rustc 1.98.0 (regression-test)\n", stderr="") + if "--sections" in command: + return subprocess.CompletedProcess(command, 0, stdout="[ 1] .text PROGBITS\n", stderr="") + if "--dyn-syms" in command: + return subprocess.CompletedProcess(command, 0, stdout="PyInit__native\n", stderr="") + raise AssertionError(f"unexpected subprocess command: {command}") + + +class _NativeModuleWithPanicHook(ModuleType): + def _panic_for_test(self) -> None: + return None def _run_verifier( @@ -64,14 +74,16 @@ def _run_verifier( *, exposes_panic: bool = False, ) -> int: - native_module: Final = ModuleType("litellm.rust_bridge._native") - if exposes_panic: - setattr(native_module, "_panic_for_test", lambda: None) + native_module: Final = ( + _NativeModuleWithPanicHook("litellm.rust_bridge._native") + if exposes_panic + else ModuleType("litellm.rust_bridge._native") + ) def _fake_load_native_module(_: Path) -> ModuleType: return native_module - environment: Final = {"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")} + environment: Final = MappingProxyType({"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")}) return verifier.main( (str(_MODULE_PATH), str(wheel)), environment, @@ -102,8 +114,8 @@ def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None: @pytest.mark.parametrize( "metadata_tags", - [None, ("cp312-cp312-linux_x86_64",)], - ids=["missing", "mismatched"], + (None, ("cp312-cp312-linux_x86_64",)), + ids=("missing", "mismatched"), ) def test_rejects_missing_or_mismatched_wheel_metadata_tag( tmp_path: Path, From 90eadac40927315831bcc7d7ae6de57f155acbd8 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 06:55:19 -0700 Subject: [PATCH 50/67] test(build): keep wheel checks outside package --- .../scripts}/smoke_test_native_wheel.py | 0 .../scripts}/verify_linux_native_wheel.py | 0 .github/workflows/test-rust.yml | 12 +++--- .../test_verify_linux_native_wheel.py | 38 +++++++++++++++++-- 4 files changed, 41 insertions(+), 9 deletions(-) rename {litellm/rust_bridge => .github/scripts}/smoke_test_native_wheel.py (100%) rename {litellm/rust_bridge => .github/scripts}/verify_linux_native_wheel.py (100%) diff --git a/litellm/rust_bridge/smoke_test_native_wheel.py b/.github/scripts/smoke_test_native_wheel.py similarity index 100% rename from litellm/rust_bridge/smoke_test_native_wheel.py rename to .github/scripts/smoke_test_native_wheel.py diff --git a/litellm/rust_bridge/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py similarity index 100% rename from litellm/rust_bridge/verify_linux_native_wheel.py rename to .github/scripts/verify_linux_native_wheel.py diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 271c73733a3..aada0fcf239 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -7,8 +7,8 @@ on: - ".cargo/**" - "pyproject.toml" - "rust-toolchain.toml" - - "litellm/rust_bridge/smoke_test_native_wheel.py" - - "litellm/rust_bridge/verify_linux_native_wheel.py" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -21,8 +21,8 @@ on: - ".cargo/**" - "pyproject.toml" - "rust-toolchain.toml" - - "litellm/rust_bridge/smoke_test_native_wheel.py" - - "litellm/rust_bridge/verify_linux_native_wheel.py" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" permissions: @@ -115,9 +115,9 @@ jobs: --config-setting "maturin.build-args=--features panic-test,extension-module" - name: Smoke-test native panic unwinding - run: python litellm/rust_bridge/smoke_test_native_wheel.py panic-dist/*.whl + run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl - name: Verify stripped native extension env: RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - run: python litellm/rust_bridge/verify_linux_native_wheel.py dist/*.whl + run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl diff --git a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py index 8d0082dddc1..e449d4392d8 100644 --- a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -1,16 +1,48 @@ from __future__ import annotations +import importlib.util import subprocess +import sys import zipfile +from collections.abc import Callable, Mapping, Sequence from pathlib import Path from types import MappingProxyType, ModuleType -from typing import Final +from typing import Final, Protocol, cast import pytest -from litellm.rust_bridge import verify_linux_native_wheel as verifier -_MODULE_PATH: Final = Path(verifier.__file__) +class _CommandRunner(Protocol): + def __call__( + self, + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +class _VerifierModule(Protocol): + main: Callable[ + [ + Sequence[str] | None, + Mapping[str, str] | None, + Callable[[Path], ModuleType | None], + _CommandRunner, + ], + int, + ] + + +_REPO_ROOT: Final = Path(__file__).resolve().parents[3] +_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "verify_linux_native_wheel.py" +_SPEC: Final = importlib.util.spec_from_file_location("verify_linux_native_wheel", _MODULE_PATH) +assert _SPEC is not None and _SPEC.loader is not None +_LOADED_VERIFIER: Final = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _LOADED_VERIFIER +_SPEC.loader.exec_module(_LOADED_VERIFIER) +verifier: Final = cast(_VerifierModule, _LOADED_VERIFIER) _EXPECTED_TAG: Final = "cp310-abi3-linux_x86_64" _NATIVE_MEMBER: Final = "litellm/rust_bridge/_native.abi3.so" From 9de4e84feb128f2248253d9f52ae13049f2c88a5 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 15:07:57 -0700 Subject: [PATCH 51/67] refactor(rust): extract domain-neutral Python interop --- litellm-rust/AGENTS.md | 9 +- litellm-rust/CLAUDE.md | 9 +- litellm-rust/Cargo.lock | 109 +++++++++++++++++- litellm-rust/Cargo.toml | 3 + litellm-rust/README.md | 8 +- .../PROVIDER_CODING_STANDARDS.md | 2 +- litellm-rust/crates/ai-gateway/README.md | 7 +- .../core/tests/workspace_crate_allowlist.rs | 16 ++- litellm-rust/crates/python-bridge/AGENTS.md | 2 +- litellm-rust/crates/python-bridge/CLAUDE.md | 5 +- litellm-rust/crates/python-bridge/Cargo.toml | 3 +- .../python-bridge/benches/serialization.rs | 11 +- litellm-rust/crates/python-bridge/src/gil.rs | 32 ----- litellm-rust/crates/python-bridge/src/lib.rs | 16 +-- .../python-bridge/tests/marshal_boundary.rs | 11 +- litellm-rust/crates/python-interop/AGENTS.md | 1 + litellm-rust/crates/python-interop/Cargo.toml | 15 +++ litellm-rust/crates/python-interop/src/gil.rs | 21 ++++ litellm-rust/crates/python-interop/src/lib.rs | 5 + .../src/marshal.rs | 0 .../crates/python-interop/tests/interop.rs | 44 +++++++ 21 files changed, 247 insertions(+), 82 deletions(-) delete mode 100644 litellm-rust/crates/python-bridge/src/gil.rs create mode 100644 litellm-rust/crates/python-interop/AGENTS.md create mode 100644 litellm-rust/crates/python-interop/Cargo.toml create mode 100644 litellm-rust/crates/python-interop/src/gil.rs create mode 100644 litellm-rust/crates/python-interop/src/lib.rs rename litellm-rust/crates/{python-bridge => python-interop}/src/marshal.rs (100%) create mode 100644 litellm-rust/crates/python-interop/tests/interop.rs diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index 36a5ad5a8f4..b8b6291283d 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -1,6 +1,6 @@ # AGENTS.md -litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are MODULES inside the layers. +litellm-rust has four crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers. ## Crates @@ -8,9 +8,10 @@ litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes ( |-------|------| | litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. | | litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. ## Where a route lives @@ -28,7 +29,7 @@ core/src/messages/ Handlers never live in `ai-gateway`. `ocr`, `audio_transcription`, and `realtime` are still hosted there from before this rule; they move to `core` as they are touched. -Adding a crate: default to a MODULE. New crate ONLY on a real trigger — separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these. +Adding a crate: default to a module. A new crate requires a real trigger: separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these. Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional. diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md index fe6ceedbb86..3dcf1853efc 100644 --- a/litellm-rust/CLAUDE.md +++ b/litellm-rust/CLAUDE.md @@ -21,12 +21,13 @@ variants of it. The test for a good abstraction is that adding the next provider is a few declarative lines, not a new file of duplicated flow. Only diverge from the base when behavior is genuinely different, and say so explicitly in the PR. -## Crates (exactly three — see AGENTS.md) +## Crates (see AGENTS.md) `litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call. `litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and -`litellm-python-bridge` exposes it to the Python SDK. A crate is a **layer**, not -a route — add modules, not crates. +`litellm-python-bridge` exposes it to the Python SDK. `litellm-python-interop` +holds domain-neutral PyO3 primitives shared by Python-facing Rust code. A crate +is a layer or shared foundation, not a route; add modules, not crates. ## Core Boundary @@ -175,7 +176,7 @@ cd litellm-rust cargo fmt --check # the ai-gateway binary + server code is behind the `server` feature cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings -cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings +cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings cargo test --workspace ``` diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 4388e561026..dd41cf0e84b 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -919,6 +919,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.33" @@ -972,6 +978,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "h2" version = "0.3.27" @@ -1432,14 +1444,24 @@ dependencies = [ "criterion", "litellm-ai-gateway", "litellm-core", + "litellm-python-interop", "pyo3", "pyo3-async-runtimes", - "pythonize", - "serde", "serde_json", "tokio", ] +[[package]] +name = "litellm-python-interop" +version = "0.1.0" +dependencies = [ + "pyo3", + "pythonize", + "rstest", + "serde", + "serde_json", +] + [[package]] name = "litemap" version = "0.8.2" @@ -1627,6 +1649,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -1899,6 +1930,12 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.28" @@ -1956,6 +1993,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.119", + "unicode-ident", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -2488,6 +2554,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + [[package]] name = "tower" version = "0.5.3" @@ -2903,6 +2999,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index c17a0605fc7..c447d915abe 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/core", "crates/ai-gateway", + "crates/python-interop", "crates/python-bridge", ] resolver = "2" @@ -15,12 +16,14 @@ repository = "https://github.com/BerriAI/litellm" [workspace.dependencies] litellm-core = { path = "crates/core" } litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false } +litellm-python-interop = { path = "crates/python-interop" } axum = "0.7" pyo3 = "0.29.0" pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } pythonize = "0.29.0" rand = "0.8" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] } +rstest = "0.26.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10" diff --git a/litellm-rust/README.md b/litellm-rust/README.md index bcccf93300b..a0d79c6f0a5 100644 --- a/litellm-rust/README.md +++ b/litellm-rust/README.md @@ -26,9 +26,10 @@ coverage and production evidence. |-------|------| | litellm-core | The SDK. Per-route entrypoints (`messages::messages()`), types, provider transforms (modules under `providers/`), provider resolution, auth, the provider HTTP call, and the router. | | litellm-ai-gateway | The axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. ## Layout @@ -38,7 +39,8 @@ crates/ src/messages/ mod.rs (entrypoint), types, transformation, prepare, handler, client src/providers/anthropic/messages/transformation.rs ai-gateway/ Axum server + WebSocket hosts; calls core entrypoints. - python-bridge/ PyO3 bridge for Python LiteLLM. + python-interop/ Domain-neutral PyO3 conversion and GIL primitives. + python-bridge/ PyO3 API adapter for Python LiteLLM. ``` The folder shape follows the Python provider tree: diff --git a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md index a1860d8a9c9..4a689cb9579 100644 --- a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md +++ b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md @@ -54,6 +54,6 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages` cd litellm-rust cargo fmt --check cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings - cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings + cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings cargo test --workspace ``` diff --git a/litellm-rust/crates/ai-gateway/README.md b/litellm-rust/crates/ai-gateway/README.md index 7a6c620ee84..5cbb47220be 100644 --- a/litellm-rust/crates/ai-gateway/README.md +++ b/litellm-rust/crates/ai-gateway/README.md @@ -6,15 +6,16 @@ dials OpenAI upstream, and splices the two sockets frame-by-frame. ## Crates -`litellm-rust` is exactly three crates (a crate is a **layer**, not a route): +`litellm-rust` has four crates. A crate is a layer or shared foundation, not a route: | Crate | Role | |-------|------| | litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. | | litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. - **Client endpoint:** `wss:///v1/realtime?model=` (WebSocket) - **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset) diff --git a/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs b/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs index 656ba033b62..8a8a5ea263a 100644 --- a/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs +++ b/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs @@ -1,7 +1,8 @@ -//! Enforcement: the litellm-rust workspace has exactly three crates. +//! Enforcement: the litellm-rust workspace has exactly four crates. //! -//! `core` (pure translation), `ai-gateway` (routes + all network I/O), and -//! `python-bridge` (the PyO3 cdylib). Adding or removing a crate must be a +//! `core` (the Rust SDK), `ai-gateway` (the HTTP/WebSocket host), +//! `python-interop` (domain-neutral PyO3 primitives), and `python-bridge` (the +//! PyO3 cdylib). Adding or removing a crate must be a //! deliberate act: this test fails until the allowlist here is updated, forcing //! whoever changes the crate set to justify the new crate per the rule that a //! crate is a layer needing independent compilation / its own deps / a separate @@ -16,10 +17,15 @@ use std::path::{Path, PathBuf}; /// The one true crate set. Update BOTH this and `litellm-rust/AGENTS.md` when the /// workspace legitimately gains or loses a crate. -const EXPECTED_MEMBERS: &[&str] = &["crates/core", "crates/ai-gateway", "crates/python-bridge"]; +const EXPECTED_MEMBERS: &[&str] = &[ + "crates/core", + "crates/ai-gateway", + "crates/python-interop", + "crates/python-bridge", +]; /// The crate subdirectory names that must exist under `crates/`. -const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-bridge"]; +const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-interop", "python-bridge"]; const MISMATCH: &str = "litellm-rust crate set changed — update this allowlist AND litellm-rust/AGENTS.md, and justify the crate per the rule (crate = layer needing independent compilation / its own deps / a separate artifact)."; diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index ad3cddfa5fd..42282ca4da4 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -1,3 +1,3 @@ -litellm-python-bridge is the PyO3 cdylib that exposes Rust to the litellm Python SDK — a thin adapter (Python objects → Rust calls → Python results) over the litellm-core route entrypoints (e.g. `litellm_core::messages::messages`). +litellm-python-bridge is the PyO3 cdylib that exposes LiteLLM Rust APIs to the Python SDK. Keep API registration, domain dependency wiring, request assembly, and Python exception mapping here. Put domain-neutral Python/Serde conversion and GIL primitives in litellm-python-interop. Keep it thin: no business logic, no transforms, no I/O orchestration — just marshal in/out and call the core entrypoint. diff --git a/litellm-rust/crates/python-bridge/CLAUDE.md b/litellm-rust/crates/python-bridge/CLAUDE.md index 3ce8b8c639a..d25ae5a8130 100644 --- a/litellm-rust/crates/python-bridge/CLAUDE.md +++ b/litellm-rust/crates/python-bridge/CLAUDE.md @@ -5,8 +5,9 @@ Rules for `litellm-rust/crates/python-bridge`. ## Responsibility `python-bridge` is the PyO3 boundary between Python LiteLLM and Rust transforms. -Keep this crate thin. It adapts Python objects to Rust payloads and returns -Python-compatible dictionaries. +Keep this crate thin. It exposes LiteLLM Rust APIs, assembles domain requests, +maps domain errors to Python exceptions, and delegates generic conversion and +GIL handling to `litellm-python-interop`. ## Bridge Shape diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index b1fdfd7677a..498003de149 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -18,10 +18,9 @@ panic-test = [] [dependencies] litellm-core = { workspace = true, features = ["bedrock-auth"] } litellm-ai-gateway = { workspace = true, default-features = false } +litellm-python-interop.workspace = true pyo3.workspace = true pyo3-async-runtimes.workspace = true -pythonize.workspace = true -serde.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/python-bridge/benches/serialization.rs b/litellm-rust/crates/python-bridge/benches/serialization.rs index 8a90cf667d0..0b9436d0cb7 100644 --- a/litellm-rust/crates/python-bridge/benches/serialization.rs +++ b/litellm-rust/crates/python-bridge/benches/serialization.rs @@ -2,6 +2,7 @@ use std::hint::black_box; use std::time::Duration; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use litellm_python_interop::{from_py, to_py}; use pyo3::prelude::*; use pyo3::types::PyDict; use serde_json::{Value, json}; @@ -25,7 +26,7 @@ fn former_json_roundtrip_from_py(py: Python<'_>, value: &Bound<'_, PyAny>) -> Va } fn pythonize_from_py(value: &Bound<'_, PyAny>) -> Value { - pythonize::depythonize(value).expect("payload should depythonize") + from_py(value).expect("payload should depythonize") } fn former_json_roundtrip_to_py(py: Python<'_>, value: &Value) -> Py { @@ -37,12 +38,10 @@ fn former_json_roundtrip_to_py(py: Python<'_>, value: &Value) -> Py { } fn pythonize_to_py(py: Python<'_>, value: &Value) -> Py { - pythonize::pythonize(py, value) - .expect("response should pythonize") - .unbind() + to_py(py, value).expect("response should pythonize") } -fn serialization(c: &mut Criterion) { +fn bridge_serialization(c: &mut Criterion) { Python::initialize(); Python::attach(|py| { for &(label, payload_bytes) in PAYLOAD_SIZES { @@ -98,6 +97,6 @@ criterion_group! { .sample_size(20) .warm_up_time(Duration::from_secs(1)) .measurement_time(Duration::from_secs(4)); - targets = serialization + targets = bridge_serialization } criterion_main!(benches); diff --git a/litellm-rust/crates/python-bridge/src/gil.rs b/litellm-rust/crates/python-bridge/src/gil.rs deleted file mode 100644 index e887c8ec1e3..00000000000 --- a/litellm-rust/crates/python-bridge/src/gil.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! GIL accounting. -//! -//! A single chokepoint for releasing the GIL around blocking work. Every -//! blocking call in the bridge goes through [`release_gil`] instead of calling -//! `Python::detach` directly, so the release count stays accurate and we -//! have one place to extend later (timing histograms, per-call labels, etc.). - -use std::sync::atomic::{AtomicU64, Ordering}; - -use pyo3::prelude::*; - -/// Number of times the bridge has released the GIL since process start. -static GIL_RELEASES: AtomicU64 = AtomicU64::new(0); - -/// Release the GIL around `f`, recording the release. -/// -/// `f` must not touch any Python state — that is what makes releasing the GIL -/// safe. Returning the value back to Python re-acquires the GIL at the call -/// site, after `f` has finished. -pub fn release_gil(py: Python<'_>, f: F) -> T -where - F: FnOnce() -> T + Send, - T: Send, -{ - GIL_RELEASES.fetch_add(1, Ordering::Relaxed); - py.detach(f) -} - -/// Total GIL releases performed by the bridge so far. -pub fn release_count() -> u64 { - GIL_RELEASES.load(Ordering::Relaxed) -} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 18e0b05cbb5..746e0770f9b 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -13,16 +13,12 @@ use litellm_core::chat_completions::{ use litellm_core::error::CoreError; use litellm_core::messages::messages as run_messages; use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; +use litellm_python_interop::{from_py, release_count, release_gil, to_py}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyAny, PyDict}; use serde_json::{Map, Value}; -mod gil; -mod marshal; - -use marshal::{from_py, to_py}; - pyo3::create_exception!( _native, RustBridgeDeclined, @@ -230,7 +226,7 @@ fn ocr( timeout_seconds, )?; - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_ocr(OcrRequest { model: &model, document, @@ -318,7 +314,7 @@ fn transcription( }; let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; let timeout = optional_timeout(timeout_seconds); - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_audio_transcription( AudioTranscriptionRequest { model: &model, @@ -419,7 +415,7 @@ fn messages( let (body, extra_headers, timeout) = marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_messages(MessagesRequest { model: &model, body, @@ -546,7 +542,7 @@ fn chat_completions( timeout_seconds, )?; - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_chat_completions( ChatCompletionsRequest { model: &model, @@ -610,7 +606,7 @@ fn achat_completions( #[pyfunction] fn gil_stats(py: Python<'_>) -> PyResult> { let stats = PyDict::new(py); - stats.set_item("releases", gil::release_count())?; + stats.set_item("releases", release_count())?; Ok(stats.into_any().unbind()) } diff --git a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs index 6a6ede22e85..d397d20b9fd 100644 --- a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs +++ b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs @@ -1,7 +1,7 @@ use std::fs; use std::path::{Path, PathBuf}; -const DISALLOWED_OUTSIDE_MARSHAL: &[&str] = &[ +const DISALLOWED_OUTSIDE_INTEROP: &[&str] = &[ "py.import(\"json\")", "pythonize::", "serde_json::to_string", @@ -33,18 +33,15 @@ fn rust_sources(directory: &Path) -> Vec { } #[test] -fn serialization_is_centralized_in_marshal_module() { +fn serialization_uses_the_interop_boundary() { let root = source_root(); for path in rust_sources(&root) { - if path == root.join("marshal.rs") { - continue; - } let source = fs::read_to_string(&path).expect("bridge source should be readable"); - for disallowed in DISALLOWED_OUTSIDE_MARSHAL { + for disallowed in DISALLOWED_OUTSIDE_INTEROP { assert!( !source.contains(disallowed), - "{} bypasses the typed marshal module with `{disallowed}`", + "{} bypasses litellm-python-interop with `{disallowed}`", path.display() ); } diff --git a/litellm-rust/crates/python-interop/AGENTS.md b/litellm-rust/crates/python-interop/AGENTS.md new file mode 100644 index 00000000000..d1d61e5dfa0 --- /dev/null +++ b/litellm-rust/crates/python-interop/AGENTS.md @@ -0,0 +1 @@ +litellm-python-interop is the domain-neutral PyO3 foundation. Keep generic Python/Serde conversion and interpreter primitives here. Do not add LiteLLM domain crates, route types, API registration, or cdylib build features. diff --git a/litellm-rust/crates/python-interop/Cargo.toml b/litellm-rust/crates/python-interop/Cargo.toml new file mode 100644 index 00000000000..9da6af6e2e2 --- /dev/null +++ b/litellm-rust/crates/python-interop/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "litellm-python-interop" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +pyo3.workspace = true +pythonize.workspace = true +serde.workspace = true + +[dev-dependencies] +rstest.workspace = true +serde_json.workspace = true diff --git a/litellm-rust/crates/python-interop/src/gil.rs b/litellm-rust/crates/python-interop/src/gil.rs new file mode 100644 index 00000000000..04b966a6002 --- /dev/null +++ b/litellm-rust/crates/python-interop/src/gil.rs @@ -0,0 +1,21 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use pyo3::prelude::*; + +static GIL_RELEASES: AtomicU64 = AtomicU64::new(0); + +/// Runs work detached from the interpreter and records the release. +/// +/// `f` must not access Python state while the interpreter is detached. +pub fn release_gil(py: Python<'_>, f: F) -> T +where + F: FnOnce() -> T + Send, + T: Send, +{ + GIL_RELEASES.fetch_add(1, Ordering::Relaxed); + py.detach(f) +} + +pub fn release_count() -> u64 { + GIL_RELEASES.load(Ordering::Relaxed) +} diff --git a/litellm-rust/crates/python-interop/src/lib.rs b/litellm-rust/crates/python-interop/src/lib.rs new file mode 100644 index 00000000000..df2bd260fdb --- /dev/null +++ b/litellm-rust/crates/python-interop/src/lib.rs @@ -0,0 +1,5 @@ +mod gil; +mod marshal; + +pub use gil::{release_count, release_gil}; +pub use marshal::{from_py, to_py}; diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-interop/src/marshal.rs similarity index 100% rename from litellm-rust/crates/python-bridge/src/marshal.rs rename to litellm-rust/crates/python-interop/src/marshal.rs diff --git a/litellm-rust/crates/python-interop/tests/interop.rs b/litellm-rust/crates/python-interop/tests/interop.rs new file mode 100644 index 00000000000..9c456dcb938 --- /dev/null +++ b/litellm-rust/crates/python-interop/tests/interop.rs @@ -0,0 +1,44 @@ +use pyo3::Python; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; + +use litellm_python_interop::{from_py, release_count, release_gil, to_py}; + +struct InitializedPython; + +impl InitializedPython { + fn attach(&self, f: F) -> R + where + F: for<'py> FnOnce(Python<'py>) -> R, + { + Python::attach(f) + } +} + +#[fixture] +#[once] +fn initialized_python() -> InitializedPython { + Python::initialize(); + InitializedPython +} + +#[rstest] +fn serde_values_round_trip_through_python(#[from(initialized_python)] python: &InitializedPython) { + python.attach(|py| { + let expected = json!({"model": "test", "items": [1, true, null]}); + let python_value = to_py(py, &expected).expect("value should convert to Python"); + let actual: Value = + from_py(python_value.bind(py)).expect("Python value should convert to serde"); + + assert_eq!(actual, expected); + }); +} + +#[rstest] +fn release_gil_runs_work_and_records_it(#[from(initialized_python)] python: &InitializedPython) { + let before = release_count(); + let result = python.attach(|py| release_gil(py, || 42)); + + assert_eq!(result, 42); + assert_eq!(release_count(), before + 1); +} From 518a2a70f159c1e118d1ea0d387c7da8ba87d642 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 05:47:01 -0700 Subject: [PATCH 52/67] refactor(rust): standardize the core Error type --- .../src/audio_transcription/common_utils.rs | 10 +-- .../src/audio_transcription/handler.rs | 31 ++++---- .../src/audio_transcription/hooks.rs | 58 +++++++------- .../ai-gateway/src/audio_transcription/mod.rs | 4 +- .../crates/ai-gateway/src/io/realtime.rs | 43 +++++------ .../crates/ai-gateway/src/io/realtime_pool.rs | 4 +- .../crates/ai-gateway/src/io/responses_ws.rs | 76 +++++++++---------- .../crates/ai-gateway/src/ocr/common_utils.rs | 73 +++++++++--------- .../crates/ai-gateway/src/ocr/handler.rs | 15 ++-- .../crates/ai-gateway/src/ocr/hooks.rs | 65 ++++++++-------- litellm-rust/crates/ai-gateway/src/ocr/mod.rs | 4 +- .../crates/ai-gateway/src/ocr/tests.rs | 8 +- .../crates/ai-gateway/src/python/config.rs | 12 ++- .../ai-gateway/src/routes/messages/mod.rs | 38 +++++----- .../ai-gateway/src/routes/messages/service.rs | 16 ++-- .../ai-gateway/src/routes/realtime/service.rs | 11 ++- .../src/routes/responses/service.rs | 12 +-- .../src/audio_transcription/transformation.rs | 11 ++- .../crates/core/src/call_lifecycle/mod.rs | 34 ++++----- .../core/src/chat_completions/common_utils.rs | 7 +- .../core/src/chat_completions/handler.rs | 30 ++++---- .../crates/core/src/chat_completions/mod.rs | 5 +- .../core/src/chat_completions/prepare.rs | 21 +++-- .../crates/core/src/chat_completions/tests.rs | 40 +++++----- .../src/chat_completions/transformation.rs | 11 ++- litellm-rust/crates/core/src/error.rs | 8 +- litellm-rust/crates/core/src/http_utils.rs | 8 +- litellm-rust/crates/core/src/lib.rs | 2 +- .../crates/core/src/messages/common_utils.rs | 7 +- .../crates/core/src/messages/handler.rs | 25 +++--- litellm-rust/crates/core/src/messages/mod.rs | 7 +- .../crates/core/src/messages/prepare.rs | 12 +-- .../crates/core/src/messages/tests.rs | 10 +-- .../core/src/messages/transformation.rs | 11 ++- .../crates/core/src/ocr/transformation.rs | 11 ++- .../anthropic/chat_completions/tests.rs | 16 ++-- .../chat_completions/transformation.rs | 25 +++--- .../anthropic/messages/transformation.rs | 12 +-- .../azure_ai/messages/transformation.rs | 22 +++--- .../providers/azure_ai/ocr/transformation.rs | 62 +++++++-------- .../providers/bedrock/audio_transcription.rs | 20 ++--- .../core/src/providers/bedrock/aws_base.rs | 38 +++++----- .../bedrock/chat_completions/tests.rs | 14 ++-- .../chat_completions/transformation.rs | 23 +++--- .../providers/mistral/ocr/transformation.rs | 28 +++---- .../openai/realtime/transformation.rs | 10 +-- .../openai/responses/transformation.rs | 6 +- .../providers/vertex_ai/ocr/transformation.rs | 50 ++++++------ .../core/src/realtime/transformation.rs | 6 +- .../core/src/responses/instrumentation.rs | 8 +- .../crates/core/src/responses/websocket.rs | 6 +- litellm-rust/crates/python-bridge/src/lib.rs | 36 ++++----- 52 files changed, 544 insertions(+), 578 deletions(-) diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs index 270d5c2d97a..140bc8aeea8 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs @@ -1,10 +1,8 @@ -use std::collections::BTreeMap; - -use litellm_core::CoreResult; use litellm_core::audio_transcription::transformation::AudioTranscriptionProviderConfig; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG; use serde_json::{Map, Value}; +use std::collections::BTreeMap; pub(super) fn audio_transcription_provider_config( provider: &str, @@ -17,7 +15,7 @@ pub(super) fn audio_transcription_provider_config( pub(super) fn string_headers( headers: Option>, -) -> CoreResult> { +) -> Result, Error> { headers .unwrap_or_default() .into_iter() @@ -26,7 +24,7 @@ pub(super) fn string_headers( .as_str() .map(|value| (key.clone(), value.to_string())) .ok_or_else(|| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "audio transcription extra_headers.{key} must be a string" )) }) diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs index 33c13550f58..1bdd4ae72a2 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs @@ -1,11 +1,9 @@ -use std::time::SystemTime; - -use litellm_core::CoreResult; use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::providers::bedrock::audio_transcription::aws_auth_config; use litellm_core::providers::bedrock::aws_base::{resolve_credentials, sign_bedrock_post}; use serde_json::Value; +use std::time::SystemTime; use super::common_utils::truncate_error_body; use super::types::ProviderAudioTranscriptionRequest; @@ -13,10 +11,9 @@ use crate::client::http_client; pub(crate) async fn execute_audio_transcription_provider_call( request: ProviderAudioTranscriptionRequest, -) -> CoreResult { - let body = serde_json::to_vec(&request.body).map_err(|error| { - CoreError::InvalidRequest(format!("invalid audio request body: {error}")) - })?; +) -> Result { + let body = serde_json::to_vec(&request.body) + .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?; let mut request_builder = http_client().post(&request.url).body(body.clone()); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -27,21 +24,20 @@ pub(crate) async fn execute_audio_transcription_provider_call( let response = request_builder .send() .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; let status = response.status(); let text = response .text() .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } - let response_json: Value = serde_json::from_str(&text).map_err(|error| { - CoreError::InvalidResponse(format!("invalid audio response JSON: {error}")) - })?; + let response_json: Value = serde_json::from_str(&text) + .map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?; Ok(request .config .transform_transcription_response(&request.model, response_json)? @@ -51,14 +47,13 @@ pub(crate) async fn execute_audio_transcription_provider_call( pub(crate) async fn sign_request( request: &ProviderAudioTranscriptionRequest, optional_params: &serde_json::Map, -) -> CoreResult { +) -> Result { let env_lookup = environment_lookup; let auth = request .config .auth_strategy(&request.model, optional_params, &env_lookup)?; - let body = serde_json::to_vec(&request.body).map_err(|error| { - CoreError::InvalidRequest(format!("invalid audio request body: {error}")) - })?; + let body = serde_json::to_vec(&request.body) + .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?; let mut headers = super::common_utils::string_headers(None)?; headers.insert("Content-Type".to_string(), "application/json".to_string()); headers.extend(request.upstream_headers.iter().cloned()); diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs index 0c9faeda6e7..5e1240de759 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs @@ -1,11 +1,9 @@ -use std::future::Future; -use std::pin::Pin; - -use litellm_core::CoreResult; use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth; use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use serde_json::{Map, Value, json}; +use std::future::Future; +use std::pin::Pin; use super::common_utils::{audio_transcription_provider_config, has_header, string_headers}; use super::handler::sign_request; @@ -26,7 +24,7 @@ pub(crate) struct AudioTranscriptionLifecycleHooks { request_metadata: RequestMetadata, } -type AudioFuture<'a, T> = Pin> + Send + 'a>>; +type AudioFuture<'a, T> = Pin> + Send + 'a>>; type AudioLogFuture<'a> = Pin + Send + 'a>>; impl AudioTranscriptionLifecycleHooks { @@ -45,7 +43,7 @@ impl AudioTranscriptionLifecycleHooks { async fn run_pre_call_guardrails( &self, request: PreparedAudioTranscriptionRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -63,17 +61,17 @@ impl AudioTranscriptionLifecycleHooks { .await .map_err(guardrail_error_to_core_error)?; let Value::Object(mut data) = guardrail_request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription pre_call guardrail must return an object".to_string(), )); }; let audio = data.remove("audio").ok_or_else(|| { - CoreError::InvalidRequest("audio transcription guardrail removed audio".to_string()) + Error::InvalidRequest("audio transcription guardrail removed audio".to_string()) })?; let optional_params = match data.remove("optional_params") { Some(Value::Object(value)) => value, Some(_) => { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription optional_params must be an object".to_string(), )); } @@ -89,9 +87,9 @@ impl AudioTranscriptionLifecycleHooks { async fn prepare_provider_request( &self, request: PreparedAudioTranscriptionRequest, - ) -> CoreResult { + ) -> Result { let config = audio_transcription_provider_config(&request.custom_llm_provider) - .ok_or_else(|| CoreError::InvalidProvider(request.custom_llm_provider.clone()))?; + .ok_or_else(|| Error::InvalidProvider(request.custom_llm_provider.clone()))?; let env_lookup = super::handler::environment_lookup; let headers = string_headers(request.extra_headers)?; let url = config.complete_url( @@ -135,7 +133,7 @@ impl AudioTranscriptionLifecycleHooks { async fn run_during_call_guardrails( &self, request: ProviderAudioTranscriptionRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -153,12 +151,12 @@ impl AudioTranscriptionLifecycleHooks { .await .map_err(guardrail_error_to_core_error)?; let Value::Object(mut data) = guardrail_request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription during_call guardrail must return an object".to_string(), )); }; let body = data.remove("body").ok_or_else(|| { - CoreError::InvalidRequest("audio transcription guardrail removed body".to_string()) + Error::InvalidRequest("audio transcription guardrail removed body".to_string()) })?; Ok(ProviderAudioTranscriptionRequest { body, ..request }) } @@ -241,7 +239,7 @@ impl CallLifecycleHooks( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -281,22 +279,22 @@ fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { } } -fn guardrail_error_to_core_error(error: GuardrailError) -> CoreError { - CoreError::InvalidRequest(format!("{}: {}", error.kind, error.message)) +fn guardrail_error_to_core_error(error: GuardrailError) -> Error { + Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) } -fn core_error_kind(error: &CoreError) -> &'static str { +fn core_error_kind(error: &Error) -> &'static str { match error { - CoreError::Auth(_) => "AuthError", - CoreError::InvalidProvider(_) => "InvalidProvider", - CoreError::InvalidRequest(_) => "InvalidRequest", - CoreError::InvalidType { .. } => "InvalidType", - CoreError::MissingField(_) => "MissingField", - CoreError::Http { .. } => "HttpError", - CoreError::InvalidResponse(_) => "InvalidResponse", - CoreError::Network(_) => "NetworkError", - CoreError::Connect(_) => "ConnectError", - CoreError::Routing(_) => "RoutingError", - CoreError::Unsupported(_) => "UnsupportedRequest", + Error::Auth(_) => "AuthError", + Error::InvalidProvider(_) => "InvalidProvider", + Error::InvalidRequest(_) => "InvalidRequest", + Error::InvalidType { .. } => "InvalidType", + Error::MissingField(_) => "MissingField", + Error::Http { .. } => "HttpError", + Error::InvalidResponse(_) => "InvalidResponse", + Error::Network(_) => "NetworkError", + Error::Connect(_) => "ConnectError", + Error::Routing(_) => "RoutingError", + Error::Unsupported(_) => "UnsupportedRequest", } } diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs index 5d33d912c40..3983846d7b6 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs @@ -1,4 +1,4 @@ -use litellm_core::CoreResult; +use litellm_core::Error; use litellm_core::call_lifecycle::CallLifecycle; use serde_json::Value; @@ -13,7 +13,7 @@ pub use types::AudioTranscriptionRequest; use handler::execute_audio_transcription_provider_call; use prepare::{PreparedAudioTranscriptionCall, prepare_audio_transcription_call}; -pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> CoreResult { +pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { let PreparedAudioTranscriptionCall { request, hooks } = prepare_audio_transcription_call(request); CallLifecycle::default() diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime.rs b/litellm-rust/crates/ai-gateway/src/io/realtime.rs index 845e7bf9527..662f7328982 100644 --- a/litellm-rust/crates/ai-gateway/src/io/realtime.rs +++ b/litellm-rust/crates/ai-gateway/src/io/realtime.rs @@ -15,8 +15,7 @@ use std::time::Duration; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{Sink, SinkExt, Stream, StreamExt}; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::realtime::transformation::RealtimeProviderConfig; use litellm_core::realtime::types::RealtimeEvent; use tokio::net::TcpStream; @@ -48,7 +47,7 @@ pub(crate) type UpstreamRx = SplitStream; /// Resolve the OpenAI API key from the explicit param or the environment. /// /// Blank/whitespace values are treated as absent (guard at resolution time). -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { +pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) @@ -58,7 +57,7 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { .ok() .filter(|key| !key.trim().is_empty()) }) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } /// Open the upstream WebSocket to OpenAI for `(model, api_key, api_base)`. @@ -70,24 +69,24 @@ pub(crate) async fn dial_upstream( model: &str, api_key: &str, api_base: Option<&str>, -) -> CoreResult { +) -> Result { let url = OPENAI_REALTIME_CONFIG.complete_url(api_base, model); let mut request = url .as_str() .into_client_request() - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; // GA realtime: only Authorization. The legacy OpenAI-Beta header triggers // beta_api_shape_disabled, so we do not send it. request.headers_mut().insert( AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|err| CoreError::Auth(err.to_string()))?, + .map_err(|err| Error::Auth(err.to_string()))?, ); let (upstream, _response) = connect_async(request) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; Ok(upstream) } @@ -96,22 +95,22 @@ pub(crate) async fn dial_upstream( /// Used by the pool to pre-read OpenAI's unprompted `session.created`. Returns an /// error on a non-text frame, a closed socket, or undecodable JSON so the pool can /// discard a misbehaving socket rather than warm it. -pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> CoreResult { +pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> Result { loop { let message = upstream_rx .next() .await - .ok_or_else(|| CoreError::Network("upstream closed before first event".to_string()))? - .map_err(|err| CoreError::Network(err.to_string()))?; + .ok_or_else(|| Error::Network("upstream closed before first event".to_string()))? + .map_err(|err| Error::Network(err.to_string()))?; match message { Message::Text(text) => { return serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(err.to_string())); + .map_err(|err| Error::InvalidResponse(err.to_string())); } // Ignore protocol frames (ping/pong) while waiting for the first event. Message::Ping(_) | Message::Pong(_) => continue, Message::Close(_) => { - return Err(CoreError::Network( + return Err(Error::Network( "upstream closed before first event".to_string(), )); } @@ -139,7 +138,7 @@ pub(crate) async fn splice( mut observe: impl FnMut(&RealtimeEvent) + Send, mut client_in: In, mut client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -154,7 +153,7 @@ where client_out .send(outbound) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } @@ -175,26 +174,26 @@ where // inflate its own spend log. Logging observes upstream events only. for outbound in config.transform_realtime_request(&event, model)?.events { let payload = serde_json::to_string(&outbound) - .map_err(|err| CoreError::InvalidResponse(err.to_string()))?; + .map_err(|err| Error::InvalidResponse(err.to_string()))?; upstream_tx .send(Message::Text(payload)) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } // upstream -> client upstream_message = upstream_rx.next() => { let Some(message) = upstream_message else { break }; // upstream closed - match message.map_err(|err| CoreError::Network(err.to_string()))? { + match message.map_err(|err| Error::Network(err.to_string()))? { Message::Text(text) => { let event: RealtimeEvent = serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(err.to_string()))?; + .map_err(|err| Error::InvalidResponse(err.to_string()))?; observe(&event); for outbound in config.transform_realtime_response(&event, model)?.events { client_out .send(outbound) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } Message::Close(_) => break, @@ -225,7 +224,7 @@ pub async fn realtime( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -258,7 +257,7 @@ pub async fn realtime_warm( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs b/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs index 4a1a3cd1166..49e9c459a88 100644 --- a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs +++ b/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs @@ -28,7 +28,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use futures_util::StreamExt; -use litellm_core::CoreResult; +use litellm_core::Error; use litellm_core::realtime::types::RealtimeEvent; use crate::io::realtime::{ @@ -438,7 +438,7 @@ impl RealtimePool { /// /// `key.api_key` is already resolved (non-blank). The first frame OpenAI sends /// unprompted is `session.created`; we buffer exactly that and read nothing more. -async fn warm_one(key: &UpstreamKey) -> CoreResult { +async fn warm_one(key: &UpstreamKey) -> Result { let upstream: UpstreamWs = dial_upstream(&key.model, &key.api_key, key.api_base.as_deref()).await?; let (tx, mut rx) = upstream.split(); diff --git a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs index 9b51019f4bc..0b01747b1a5 100644 --- a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs +++ b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs @@ -4,10 +4,10 @@ use std::time::Duration; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{Sink, SinkExt, Stream, StreamExt}; +use litellm_core::Error; use litellm_core::providers::openai::responses::transformation::OPENAI_RESPONSES_WS_CONFIG; use litellm_core::responses::types::ResponsesWsEvent; use litellm_core::responses::websocket::ResponsesWebSocketProviderConfig; -use litellm_core::{CoreError, CoreResult}; use tokio::net::TcpStream; use tokio::sync::Mutex; use tokio_tungstenite::tungstenite::Message; @@ -37,51 +37,49 @@ impl ResponsesWebSocketConnection { url: &str, headers: &HashMap, timeout: Option, - ) -> CoreResult { + ) -> Result { let mut request = url .into_client_request() - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; for (name, value) in headers { let header_name = name .parse::() - .map_err(|error| CoreError::InvalidRequest(error.to_string()))?; + .map_err(|error| Error::InvalidRequest(error.to_string()))?; let header_value = HeaderValue::from_str(value) - .map_err(|error| CoreError::InvalidRequest(error.to_string()))?; + .map_err(|error| Error::InvalidRequest(error.to_string()))?; request.headers_mut().insert(header_name, header_value); } let connect = connect_async(request); let result = match timeout { Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| { - CoreError::Network("Responses WebSocket connection timed out".to_string()) + Error::Network("Responses WebSocket connection timed out".to_string()) })?, None => connect.await, }; let (socket, _) = result.map_err(|error| match error { - tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http { + tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http { status: response.status().as_u16(), body: String::new(), }, - other => CoreError::Network(other.to_string()), + other => Error::Network(other.to_string()), })?; Ok(Self { socket: Arc::new(Mutex::new(Some(socket))), }) } - pub async fn send_text(&self, text: String) -> CoreResult<()> { + pub async fn send_text(&self, text: String) -> Result<(), Error> { let mut socket = self.socket.lock().await; let Some(socket) = socket.as_mut() else { - return Err(CoreError::Network( - "Responses WebSocket is closed".to_string(), - )); + return Err(Error::Network("Responses WebSocket is closed".to_string())); }; socket .send(Message::Text(text)) .await - .map_err(|error| CoreError::Network(error.to_string())) + .map_err(|error| Error::Network(error.to_string())) } - pub async fn recv_text(&self) -> CoreResult> { + pub async fn recv_text(&self) -> Result, Error> { let mut socket_guard = self.socket.lock().await; let Some(socket) = socket_guard.as_mut() else { return Ok(None); @@ -90,27 +88,27 @@ impl ResponsesWebSocketConnection { Some(Ok(Message::Text(text))) => Ok(Some(text)), Some(Ok(Message::Binary(bytes))) => String::from_utf8(bytes.to_vec()) .map(Some) - .map_err(|error| CoreError::InvalidResponse(error.to_string())), + .map_err(|error| Error::InvalidResponse(error.to_string())), Some(Ok(Message::Close(_))) | None => Ok(None), Some(Ok(_)) => Ok(None), - Some(Err(error)) => Err(CoreError::Network(error.to_string())), + Some(Err(error)) => Err(Error::Network(error.to_string())), } } - pub async fn close(&self) -> CoreResult<()> { + pub async fn close(&self) -> Result<(), Error> { let mut socket = self.socket.lock().await; if let Some(socket) = socket.as_mut() { socket .close(None) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } *socket = None; Ok(()) } } -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { +pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { api_key .map(str::trim) .filter(|value| !value.is_empty()) @@ -120,38 +118,38 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { .ok() .filter(|value| !value.trim().is_empty()) }) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } async fn dial_upstream( model: &str, api_key: &str, api_base: Option<&str>, -) -> CoreResult { +) -> Result { let url = OPENAI_RESPONSES_WS_CONFIG.complete_websocket_url(api_base, model); let mut request = url .as_str() .into_client_request() - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; request.headers_mut().insert( AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|error| CoreError::Auth(error.to_string()))?, + .map_err(|error| Error::Auth(error.to_string()))?, ); let result = tokio::time::timeout( Duration::from_secs(DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS), connect_async(request), ) .await - .map_err(|_| CoreError::Network("Responses WebSocket connection timed out".to_string()))?; + .map_err(|_| Error::Network("Responses WebSocket connection timed out".to_string()))?; result .map(|(socket, _)| socket) .map_err(|error| match error { - tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http { + tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http { status: response.status().as_u16(), body: String::new(), }, - other => CoreError::Network(other.to_string()), + other => Error::Network(other.to_string()), }) } @@ -166,7 +164,7 @@ impl ResponsesWebSocketStreaming { observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, - ) -> CoreResult<()> + ) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -193,7 +191,7 @@ pub(crate) async fn splice( mut observe: impl FnMut(&ResponsesWsEvent) + Send, mut client_in: In, mut client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -210,18 +208,18 @@ where .events { let payload = serde_json::to_string(&outbound) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; upstream_tx.send(Message::Text(payload)) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } message = upstream_rx.next() => { let Some(message) = message else { break }; - match message.map_err(|error| CoreError::Network(error.to_string()))? { + match message.map_err(|error| Error::Network(error.to_string()))? { Message::Text(text) => { let event = serde_json::from_str::(&text) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; observe(&event); for outbound in OPENAI_RESPONSES_WS_CONFIG .transform_ws_response(&event, model)? @@ -229,7 +227,7 @@ where { client_out.send(outbound) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } Message::Close(_) => break, @@ -252,7 +250,7 @@ pub async fn async_responses_websocket( mut observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -267,11 +265,11 @@ where .events { let payload = serde_json::to_string(&outbound) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; upstream_tx .send(Message::Text(payload)) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } ResponsesWebSocketStreaming::bidirectional_forward( @@ -296,7 +294,7 @@ pub async fn responses_ws( observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -514,7 +512,7 @@ mod tests { ) .await .expect_err("status error"); - assert!(matches!(error, CoreError::Http { status: 401, .. })); + assert!(matches!(error, Error::Http { status: 401, .. })); server.await.expect("server task"); } @@ -543,7 +541,7 @@ mod tests { ) .await .expect_err("status error"); - assert!(matches!(error, CoreError::Http { status: 500, .. })); + assert!(matches!(error, Error::Http { status: 500, .. })); server.await.expect("server task"); } } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs b/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs index 9bc2818b6e7..e0ce165dc93 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs @@ -3,8 +3,7 @@ use std::time::{Duration, Instant}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrProviderConfig; use reqwest::Url; use serde_json::{Map, Value}; @@ -56,7 +55,7 @@ fn is_azure_document_intelligence_model(model: &str) -> bool { pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { extra_headers .unwrap_or_default() .into_iter() @@ -65,7 +64,7 @@ pub(super) fn string_headers( .as_str() .map(|value| (key.clone(), value.to_string())) .ok_or_else(|| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "OCR extra_headers.{key} must be a string, got {}", litellm_core::error::json_type_name(&value) )) @@ -80,7 +79,7 @@ pub(super) fn has_header(headers: &[(String, String)], name: &str) -> bool { .any(|(key, _)| key.eq_ignore_ascii_case(name)) } -fn document_url_field(document: &Value) -> CoreResult> { +fn document_url_field(document: &Value) -> Result, Error> { let Some(object) = document.as_object() else { return Ok(None); }; @@ -138,13 +137,13 @@ fn is_blocked_ip(ip: IpAddr) -> bool { } } -fn blocked_url_error(url: &Url) -> CoreError { - CoreError::InvalidRequest(format!( +fn blocked_url_error(url: &Url) -> Error { + Error::InvalidRequest(format!( "OCR document URL rejected by SSRF protection: {url}" )) } -async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { +async fn validate_safe_fetch_url(url: &Url) -> Result<(), Error> { if !matches!(url.scheme(), "http" | "https") { return Err(blocked_url_error(url)); } @@ -162,7 +161,7 @@ async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { .ok_or_else(|| blocked_url_error(url))?; let addresses = tokio::net::lookup_host((host, port)) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let mut saw_address = false; for address in addresses { saw_address = true; @@ -176,25 +175,25 @@ async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { Ok(()) } -fn redirect_location(response: &reqwest::Response, url: &Url) -> CoreResult { +fn redirect_location(response: &reqwest::Response, url: &Url) -> Result { let location = response .headers() .get(reqwest::header::LOCATION) .and_then(|value| value.to_str().ok()) .ok_or_else(|| { - CoreError::InvalidResponse("OCR document redirect missing Location header".to_string()) + Error::InvalidResponse("OCR document redirect missing Location header".to_string()) })?; url.join(location) - .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR document redirect: {err}"))) + .map_err(|err| Error::InvalidResponse(format!("invalid OCR document redirect: {err}"))) } -async fn safe_get_document_url(url: &str) -> CoreResult<(Url, reqwest::Response)> { +async fn safe_get_document_url(url: &str) -> Result<(Url, reqwest::Response), Error> { let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .build() - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let mut current_url = Url::parse(url) - .map_err(|err| CoreError::InvalidRequest(format!("invalid OCR document URL: {err}")))?; + .map_err(|err| Error::InvalidRequest(format!("invalid OCR document URL: {err}")))?; for _ in 0..MAX_SAFE_FETCH_REDIRECTS { validate_safe_fetch_url(¤t_url).await?; @@ -202,28 +201,28 @@ async fn safe_get_document_url(url: &str) -> CoreResult<(Url, reqwest::Response) .get(current_url.clone()) .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !response.status().is_redirection() { return Ok((current_url, response)); } current_url = redirect_location(&response, ¤t_url)?; } - Err(CoreError::InvalidRequest( + Err(Error::InvalidRequest( "Too many redirects while fetching OCR document URL".to_string(), )) } -fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> CoreResult<()> { +fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Result<(), Error> { if max_bytes == 0 { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "OCR document URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}" ))); } if content_length > max_bytes { let size_mb = content_length as f64 / (1024.0 * 1024.0); let max_size_mb = max_bytes as f64 / (1024.0 * 1024.0); - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "OCR document size ({size_mb:.2}MB) exceeds maximum allowed size ({max_size_mb:.2}MB). url={url}" ))); } @@ -233,7 +232,7 @@ fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Core async fn read_response_with_limit( mut response: reqwest::Response, url: &Url, -) -> CoreResult> { +) -> Result, Error> { let max_bytes = max_document_download_bytes(); if let Some(content_length) = response.content_length() { enforce_download_size(content_length, max_bytes, url)?; @@ -246,7 +245,7 @@ async fn read_response_with_limit( while let Some(chunk) = response .chunk() .await - .map_err(|err| CoreError::Network(err.to_string()))? + .map_err(|err| Error::Network(err.to_string()))? { bytes_downloaded += chunk.len() as u64; enforce_download_size(bytes_downloaded, max_bytes, url)?; @@ -255,7 +254,7 @@ async fn read_response_with_limit( Ok(bytes) } -pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreResult { +pub(super) async fn convert_document_url_to_data_uri(document: Value) -> Result { let Some((field, url)) = document_url_field(&document)? else { return Ok(document); }; @@ -267,7 +266,7 @@ pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreRes let status = response.status(); if !status.is_success() { let body = response.text().await.unwrap_or_default(); - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&body), }); @@ -290,7 +289,7 @@ pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreRes let mut transformed = document .as_object() .cloned() - .ok_or_else(|| CoreError::InvalidRequest("OCR document must be an object".to_string()))?; + .ok_or_else(|| Error::InvalidRequest("OCR document must be an object".to_string()))?; transformed.insert(field.to_string(), Value::String(data_uri)); Ok(Value::Object(transformed)) } @@ -316,11 +315,11 @@ fn retry_after_secs(response: &reqwest::Response) -> u64 { .unwrap_or(2) } -fn operation_status(response_json: &Value) -> CoreResult<&str> { +fn operation_status(response_json: &Value) -> Result<&str, Error> { let status = response_json .get("status") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("status"))?; + .ok_or(Error::MissingField("status"))?; match status { "succeeded" => Ok("succeeded"), "running" | "notStarted" => Ok("running"), @@ -330,11 +329,11 @@ fn operation_status(response_json: &Value) -> CoreResult<&str> { .and_then(|error| error.get("message")) .and_then(Value::as_str) .unwrap_or("Unknown error"); - Err(CoreError::InvalidResponse(format!( + Err(Error::InvalidResponse(format!( "Azure Document Intelligence analysis failed: {message}" ))) } - other => Err(CoreError::InvalidResponse(format!( + other => Err(Error::InvalidResponse(format!( "Unknown operation status: {other}" ))), } @@ -345,9 +344,9 @@ pub(super) async fn poll_document_intelligence( original_url: &str, headers: &[(String, String)], timeout: Option, -) -> CoreResult { +) -> Result { if !same_origin(operation_url, original_url) { - return Err(CoreError::InvalidResponse( + return Err(Error::InvalidResponse( "Azure Document Intelligence: rejected cross-origin polling URL".to_string(), )); } @@ -358,7 +357,7 @@ pub(super) async fn poll_document_intelligence( )); loop { if start.elapsed() > timeout { - return Err(CoreError::Network(format!( + return Err(Error::Network(format!( "Azure Document Intelligence operation polling timed out after {} seconds", timeout.as_secs() ))); @@ -373,21 +372,21 @@ pub(super) async fn poll_document_intelligence( let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let retry_after = retry_after_secs(&response); let status = response.status(); let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let response_json: Value = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}")) + Error::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}")) })?; if operation_status(&response_json)? == "succeeded" { return Ok(response_json); @@ -426,7 +425,7 @@ mod tests { assert!(matches!( error, - CoreError::InvalidRequest(message) + Error::InvalidRequest(message) if message.contains("SSRF protection") )); } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs b/litellm-rust/crates/ai-gateway/src/ocr/handler.rs index 1de34eb400e..815bc84363a 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/handler.rs @@ -1,5 +1,4 @@ -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrResponseHandling; use serde_json::Value; @@ -7,7 +6,7 @@ use super::common_utils::{poll_document_intelligence, truncate_error_body}; use super::types::ProviderOcrRequest; use crate::client::http_client; -pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> CoreResult { +pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Result { let mut request_builder = http_client().post(&request.url).json(&request.body); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -19,7 +18,7 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); if request.config.response_handling() == OcrResponseHandling::AzureDocumentIntelligencePoll @@ -31,7 +30,7 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co .and_then(|value| value.to_str().ok()) .map(str::to_string) .ok_or_else(|| { - CoreError::InvalidResponse( + Error::InvalidResponse( "Azure Document Intelligence returned 202 but no Operation-Location header found" .to_string(), ) @@ -52,17 +51,17 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let response_json: Value = serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; + .map_err(|err| Error::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; Ok(request .config diff --git a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs index 95df566dc53..401e26d3b29 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs @@ -1,11 +1,9 @@ -use std::future::Future; -use std::pin::Pin; - -use litellm_core::CoreResult; use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrAuthStrategy; use serde_json::{Map, Value, json}; +use std::future::Future; +use std::pin::Pin; use super::common_utils::{ convert_document_url_to_data_uri, has_header, ocr_provider_config, string_headers, @@ -27,7 +25,7 @@ pub(crate) struct OcrLifecycleHooks { request_metadata: RequestMetadata, } -type OcrFuture<'a, T> = Pin> + Send + 'a>>; +type OcrFuture<'a, T> = Pin> + Send + 'a>>; type OcrLogFuture<'a> = Pin + Send + 'a>>; impl OcrLifecycleHooks { @@ -46,7 +44,7 @@ impl OcrLifecycleHooks { async fn run_pre_call_guardrails( &self, request: PreparedOcrRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -74,9 +72,9 @@ impl OcrLifecycleHooks { async fn prepare_provider_request( &self, request: PreparedOcrRequest, - ) -> CoreResult { + ) -> Result { let config = ocr_provider_config(&request.custom_llm_provider, &request.model) - .ok_or_else(|| CoreError::InvalidProvider(request.custom_llm_provider.clone()))?; + .ok_or_else(|| Error::InvalidProvider(request.custom_llm_provider.clone()))?; let env_lookup = |key: &str| std::env::var(key).ok(); let headers = string_headers(request.extra_headers)?; let auth_strategy = config.auth_strategy(); @@ -120,7 +118,7 @@ impl OcrLifecycleHooks { custom_llm_provider: &str, url: &str, body: Value, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(body); } @@ -217,7 +215,7 @@ impl CallLifecycleHooks for OcrLi fn async_log_failure_event<'a>( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -278,19 +276,19 @@ fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { fn parse_ocr_pre_call_guardrail_request( request: GuardrailRequest, -) -> CoreResult<(Value, Map)> { +) -> Result<(Value, Map), Error> { let Value::Object(mut data) = request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR pre_call guardrail must return an object".to_string(), )); }; let document = data.remove("document").ok_or_else(|| { - CoreError::InvalidRequest("OCR pre_call guardrail removed document".to_string()) + Error::InvalidRequest("OCR pre_call guardrail removed document".to_string()) })?; let optional_params = match data.remove("optional_params") { Some(Value::Object(params)) => params, Some(_) => { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR pre_call guardrail optional_params must be an object".to_string(), )); } @@ -299,33 +297,32 @@ fn parse_ocr_pre_call_guardrail_request( Ok((document, optional_params)) } -fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> CoreResult { +fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> Result { let Value::Object(mut data) = request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR during_call guardrail must return an object".to_string(), )); }; - data.remove("body").ok_or_else(|| { - CoreError::InvalidRequest("OCR during_call guardrail removed body".to_string()) - }) + data.remove("body") + .ok_or_else(|| Error::InvalidRequest("OCR during_call guardrail removed body".to_string())) } -fn guardrail_error_to_core_error(error: GuardrailError) -> CoreError { - CoreError::InvalidRequest(format!("{}: {}", error.kind, error.message)) +fn guardrail_error_to_core_error(error: GuardrailError) -> Error { + Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) } -fn core_error_kind(error: &CoreError) -> &'static str { +fn core_error_kind(error: &Error) -> &'static str { match error { - CoreError::Auth(_) => "AuthError", - CoreError::InvalidProvider(_) => "InvalidProvider", - CoreError::InvalidRequest(_) => "InvalidRequest", - CoreError::InvalidType { .. } => "InvalidType", - CoreError::MissingField(_) => "MissingField", - CoreError::Http { .. } => "HttpError", - CoreError::InvalidResponse(_) => "InvalidResponse", - CoreError::Network(_) => "NetworkError", - CoreError::Connect(_) => "ConnectError", - CoreError::Routing(_) => "RoutingError", - CoreError::Unsupported(_) => "UnsupportedRequest", + Error::Auth(_) => "AuthError", + Error::InvalidProvider(_) => "InvalidProvider", + Error::InvalidRequest(_) => "InvalidRequest", + Error::InvalidType { .. } => "InvalidType", + Error::MissingField(_) => "MissingField", + Error::Http { .. } => "HttpError", + Error::InvalidResponse(_) => "InvalidResponse", + Error::Network(_) => "NetworkError", + Error::Connect(_) => "ConnectError", + Error::Routing(_) => "RoutingError", + Error::Unsupported(_) => "UnsupportedRequest", } } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs index c4c13e2300c..b59ab626fd3 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs @@ -1,4 +1,4 @@ -use litellm_core::CoreResult; +use litellm_core::Error; use litellm_core::call_lifecycle::CallLifecycle; use serde_json::Value; @@ -13,7 +13,7 @@ pub use types::OcrRequest; use handler::execute_ocr_provider_call; use prepare::{PreparedOcrCall, prepare_ocr_call}; -pub async fn ocr(request: OcrRequest<'_>) -> CoreResult { +pub async fn ocr(request: OcrRequest<'_>) -> Result { let PreparedOcrCall { request, hooks } = prepare_ocr_call(request); CallLifecycle::default() .run_request(request, &hooks, execute_ocr_provider_call) diff --git a/litellm-rust/crates/ai-gateway/src/ocr/tests.rs b/litellm-rust/crates/ai-gateway/src/ocr/tests.rs index bb2a6b06501..8c3f0425149 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/tests.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/tests.rs @@ -1,7 +1,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrResponseHandling; use serde_json::{Map, Value, json}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -395,7 +395,7 @@ async fn ocr_lifecycle_runs_failure_hook_on_provider_error() { .await .expect_err("provider error propagates"); - assert!(matches!(err, CoreError::Http { status: 500, .. })); + assert!(matches!(err, Error::Http { status: 500, .. })); server.await.expect("server task completes"); assert_eq!( logger.events(), @@ -439,7 +439,7 @@ async fn ocr_lifecycle_pre_call_block_skips_provider_socket() { .await .expect_err("guardrail blocks request"); - assert!(matches!(err, CoreError::InvalidRequest(_))); + assert!(matches!(err, Error::InvalidRequest(_))); assert_eq!(guardrail.events(), vec!["async_pre_call_hook"]); assert_eq!( logger.events(), @@ -607,7 +607,7 @@ fn string_headers_rejects_non_string_values() { let err = string_headers(Some(headers)).expect_err("non-string header rejected"); assert_eq!( err, - CoreError::InvalidRequest( + Error::InvalidRequest( "OCR extra_headers.x-retry-count must be a string, got number".to_string() ) ); diff --git a/litellm-rust/crates/ai-gateway/src/python/config.rs b/litellm-rust/crates/ai-gateway/src/python/config.rs index c028d3d6b51..d5a4dd69c8d 100644 --- a/litellm-rust/crates/ai-gateway/src/python/config.rs +++ b/litellm-rust/crates/ai-gateway/src/python/config.rs @@ -6,33 +6,31 @@ //! (and recorded in [`crate::gil`]); the realtime hot path never touches Python. //! //! Compiled only under the `python-config` feature. - -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::router::{Deployment, Router}; use pyo3::prelude::*; use crate::gil; /// Load the router's `model_list` from `config_path` via the Python reader. -pub fn load_router_from_config(config_path: &str) -> CoreResult { +pub fn load_router_from_config(config_path: &str) -> Result { gil::record_acquisition(); Python::attach(|py| { let model_list = py .import("litellm.proxy.read_model_list") .and_then(|module| module.getattr("read_model_list")) .and_then(|reader| reader.call1((config_path,))) - .map_err(|err| CoreError::Routing(format!("read_model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("read_model_list failed: {err}")))?; let model_list_json: String = py .import("json") .and_then(|json| json.getattr("dumps")) .and_then(|dumps| dumps.call1((model_list,))) .and_then(|encoded| encoded.extract()) - .map_err(|err| CoreError::Routing(format!("serializing model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("serializing model_list failed: {err}")))?; let deployments: Vec = serde_json::from_str(&model_list_json) - .map_err(|err| CoreError::Routing(format!("parsing model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("parsing model_list failed: {err}")))?; Ok(Router::new(deployments)) }) diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs index 7e38d10c6ff..e9f8c477f36 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs @@ -9,7 +9,7 @@ use axum::http::StatusCode; use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE, HeaderMap, HeaderValue}; use axum::response::{IntoResponse, Response}; use axum::routing::post; -use litellm_core::CoreError; +use litellm_core::Error; use serde_json::{Map, Value}; use crate::auth::RequireMasterKey; @@ -46,7 +46,7 @@ fn stream_response(upstream: reqwest::Response) -> Result Result Result>, CoreError> { +fn forwarded_headers(headers: &HeaderMap) -> Result>, Error> { let forwarded = headers .iter() .filter(|(name, _)| { @@ -74,19 +74,19 @@ fn forwarded_headers(headers: &HeaderMap) -> Result>, }) .map(|(name, value)| { let value = value.to_str().map_err(|_| { - CoreError::InvalidRequest(format!("invalid value for header {}", name.as_str())) + Error::InvalidRequest(format!("invalid value for header {}", name.as_str())) })?; Ok((name.to_string(), Value::String(value.to_string()))) }) - .collect::, CoreError>>()?; + .collect::, Error>>()?; Ok((!forwarded.is_empty()).then_some(forwarded)) } #[derive(Debug)] -struct MessagesRouteError(CoreError); +struct MessagesRouteError(Error); -impl From for MessagesRouteError { - fn from(error: CoreError) -> Self { +impl From for MessagesRouteError { + fn from(error: Error) -> Self { Self(error) } } @@ -94,28 +94,28 @@ impl From for MessagesRouteError { impl IntoResponse for MessagesRouteError { fn into_response(self) -> Response { let (status, message) = match self.0 { - CoreError::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message), - CoreError::InvalidProvider(_) | CoreError::Routing(_) => ( + Error::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message), + Error::InvalidProvider(_) | Error::Routing(_) => ( StatusCode::NOT_FOUND, "no messages deployment is configured for this model".to_string(), ), - CoreError::Auth(_) => ( + Error::Auth(_) => ( StatusCode::BAD_GATEWAY, "messages provider authentication failed".to_string(), ), - CoreError::Http { .. } - | CoreError::Network(_) - | CoreError::Connect(_) - | CoreError::InvalidResponse(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) => ( + Error::Http { .. } + | Error::Network(_) + | Error::Connect(_) + | Error::InvalidResponse(_) + | Error::InvalidType { .. } + | Error::MissingField(_) => ( StatusCode::BAD_GATEWAY, "messages provider request failed".to_string(), ), // The gateway has no Python implementation to decline to, so a // request the core cannot serve is reported to the caller. The // reason is a fixed internal string, never provider content. - CoreError::Unsupported(reason) => ( + Error::Unsupported(reason) => ( StatusCode::BAD_REQUEST, format!("messages request is not supported: {reason}"), ), diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs index 5f4c5fe8de4..4fd29db05d6 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs @@ -1,10 +1,10 @@ use std::sync::Arc; +use litellm_core::Error; use litellm_core::constants::ANTHROPIC_MESSAGES_PROVIDER; use litellm_core::messages::types::MessagesRequest; use litellm_core::messages::{messages, messages_stream}; use litellm_core::router::Router; -use litellm_core::{CoreError, CoreResult}; use serde_json::{Map, Value}; pub(crate) enum MessagesResponse { @@ -16,16 +16,16 @@ pub async fn run( router: &Arc, body: Value, extra_headers: Option>, -) -> CoreResult { +) -> Result { let model = body .get("model") .and_then(Value::as_str) .map(str::trim) .filter(|model| !model.is_empty()) - .ok_or_else(|| CoreError::InvalidRequest("messages body requires a model".to_string()))?; - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + .ok_or_else(|| Error::InvalidRequest("messages body requires a model".to_string()))?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let provider_model = deployment.litellm_params.model.as_str(); let upstream_model = provider_model .split_once('/') @@ -37,7 +37,7 @@ pub async fn run( }; let mut body = body; body.as_object_mut() - .ok_or_else(|| CoreError::InvalidRequest("messages body must be an object".to_string()))? + .ok_or_else(|| Error::InvalidRequest("messages body must be an object".to_string()))? .insert( "model".to_string(), Value::String(upstream_model.to_string()), @@ -60,6 +60,6 @@ pub async fn run( serde_json::to_value(response) .map(MessagesResponse::Json) .map_err(|err| { - CoreError::InvalidResponse(format!("failed to serialize messages response: {err}")) + Error::InvalidResponse(format!("failed to serialize messages response: {err}")) }) } diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs index 4ae8cfe7379..b8ee77c4269 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs @@ -11,8 +11,7 @@ use std::time::Duration; use crate::io::realtime_pool::{RealtimePool, upstream_key}; use futures_util::{Sink, Stream}; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::realtime::types::RealtimeEvent; use litellm_core::router::Router; @@ -29,15 +28,15 @@ pub async fn run( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, >::Error: std::fmt::Display, { - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let params = &deployment.litellm_params; // Strip a leading `openai/` so the OpenAI-only realtime fn gets the bare model. let provider_model = params diff --git a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs b/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs index 165c95695d3..e8f840c0c8e 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs @@ -2,13 +2,13 @@ use std::sync::Arc; use std::time::Duration; use futures_util::{Sink, Stream}; +use litellm_core::Error; use litellm_core::call_lifecycle::{CallLifecycle, CallLifecycleContext}; use litellm_core::responses::instrumentation::{ ResponsesWsCallbackPayload, ResponsesWsInstrumentation, ResponsesWsLogOutcome, ResponsesWsMetadata, }; use litellm_core::responses::types::ResponsesWsEvent; -use litellm_core::{CoreError, CoreResult}; use crate::integrations::custom_logger::{ CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, ModelCallDetails, @@ -26,22 +26,22 @@ pub async fn run( metadata: RequestMetadata, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, Out::Error: std::fmt::Display, { - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let params = &deployment.litellm_params; let provider_model = params .model .strip_prefix("openai/") .unwrap_or(¶ms.model); if params.model.contains('/') && !params.model.starts_with("openai/") { - return Err(CoreError::InvalidProvider( + return Err(Error::InvalidProvider( "Responses WebSocket route supports OpenAI deployments only".to_string(), )); } diff --git a/litellm-rust/crates/core/src/audio_transcription/transformation.rs b/litellm-rust/crates/core/src/audio_transcription/transformation.rs index eab34c13843..16a28fbcac0 100644 --- a/litellm-rust/crates/core/src/audio_transcription/transformation.rs +++ b/litellm-rust/crates/core/src/audio_transcription/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::CoreResult; - use super::types::{AudioTranscriptionRequestData, AudioTranscriptionResponseData}; #[derive(Clone, Debug, PartialEq, Eq)] @@ -32,13 +31,13 @@ pub trait AudioTranscriptionProviderConfig: Sync { model: &str, audio: Value, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_transcription_response( &self, model: &str, response_json: Value, - ) -> CoreResult; + ) -> Result; fn complete_url( &self, @@ -46,12 +45,12 @@ pub trait AudioTranscriptionProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth_strategy( &self, model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; } diff --git a/litellm-rust/crates/core/src/call_lifecycle/mod.rs b/litellm-rust/crates/core/src/call_lifecycle/mod.rs index d9b68a1b726..637c156e192 100644 --- a/litellm-rust/crates/core/src/call_lifecycle/mod.rs +++ b/litellm-rust/crates/core/src/call_lifecycle/mod.rs @@ -1,7 +1,7 @@ use std::future::Future; use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use crate::{CoreError, CoreResult}; +use crate::Error; pub mod types; @@ -11,14 +11,14 @@ pub use types::{ }; pub trait CallLifecycleHooks: Send + Sync { - type PreCallFuture<'a>: Future> + Send + 'a + type PreCallFuture<'a>: Future> + Send + 'a where Self: 'a, InitialReq: 'a, ProviderReq: 'a, Resp: 'a; - type DuringCallFuture<'a>: Future> + Send + 'a + type DuringCallFuture<'a>: Future> + Send + 'a where Self: 'a, InitialReq: 'a, @@ -56,7 +56,7 @@ pub trait CallLifecycleHooks: Send + Sync { fn async_log_failure_event<'a>( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a>; } @@ -86,12 +86,12 @@ impl<'a> CallLifecycle<'a> { request: InitialReq, hooks: &Hooks, provider_call: ProviderCall, - ) -> CoreResult + ) -> Result where InitialReq: CallLifecycleRequest, Hooks: CallLifecycleHooks, ProviderCall: FnOnce(ProviderReq) -> ProviderFuture, - ProviderFuture: Future>, + ProviderFuture: Future>, { let context = request.lifecycle_context(); self.run(context, request, hooks, provider_call).await @@ -103,11 +103,11 @@ impl<'a> CallLifecycle<'a> { request: InitialReq, hooks: &Hooks, provider_call: ProviderCall, - ) -> CoreResult + ) -> Result where Hooks: CallLifecycleHooks, ProviderCall: FnOnce(ProviderReq) -> ProviderFuture, - ProviderFuture: Future>, + ProviderFuture: Future>, { let call_start = epoch_seconds(); let mut phases = Vec::new(); @@ -166,7 +166,7 @@ impl<'a> CallLifecycle<'a> { &self, context: &CallLifecycleContext, hooks: &Hooks, - error: &CoreError, + error: &Error, call_start: f64, phases: &mut Vec, ) where @@ -251,8 +251,8 @@ mod tests { } impl CallLifecycleHooks for RecordingHooks { - type PreCallFuture<'a> = BoxFuture<'a, CoreResult>; - type DuringCallFuture<'a> = BoxFuture<'a, CoreResult>; + type PreCallFuture<'a> = BoxFuture<'a, Result>; + type DuringCallFuture<'a> = BoxFuture<'a, Result>; type SuccessFuture<'a> = BoxFuture<'a, ()>; type FailureFuture<'a> = BoxFuture<'a, ()>; @@ -294,7 +294,7 @@ mod tests { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -304,8 +304,8 @@ mod tests { } impl CallLifecycleHooks for RecordingHooks { - type PreCallFuture<'a> = BoxFuture<'a, CoreResult>; - type DuringCallFuture<'a> = BoxFuture<'a, CoreResult>; + type PreCallFuture<'a> = BoxFuture<'a, Result>; + type DuringCallFuture<'a> = BoxFuture<'a, Result>; type SuccessFuture<'a> = BoxFuture<'a, ()>; type FailureFuture<'a> = BoxFuture<'a, ()>; @@ -345,7 +345,7 @@ mod tests { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -383,13 +383,13 @@ mod tests { "request".to_string(), &hooks, |_request| async move { - Err::(CoreError::Network("provider down".to_string())) + Err::(Error::Network("provider down".to_string())) }, ) .await .expect_err("call fails"); - assert_eq!(error, CoreError::Network("provider down".to_string())); + assert_eq!(error, Error::Network("provider down".to_string())); assert_eq!(hooks.events(), vec!["pre_call", "during_call", "failure"]); } diff --git a/litellm-rust/crates/core/src/chat_completions/common_utils.rs b/litellm-rust/crates/core/src/chat_completions/common_utils.rs index 36eaf242a5a..ca51471eb7c 100644 --- a/litellm-rust/crates/core/src/chat_completions/common_utils.rs +++ b/litellm-rust/crates/core/src/chat_completions/common_utils.rs @@ -1,8 +1,7 @@ -use serde_json::{Map, Value}; - -use crate::error::CoreResult; +use crate::Error; use crate::http_utils::string_headers as shared_string_headers; use crate::providers::anthropic::chat_completions::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG; +use serde_json::{Map, Value}; use super::transformation::ChatCompletionsProviderConfig; @@ -23,6 +22,6 @@ pub(super) fn chat_completions_provider_config( pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { shared_string_headers(HEADER_CONTEXT, extra_headers) } diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index afc4529fd26..7e2731442cc 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,6 +1,6 @@ use serde_json::Value; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::http_utils::truncate_error_body; use super::client::http_client; @@ -11,9 +11,9 @@ use super::types::{ pub(super) async fn execute_chat_completions_provider_call( request: ProviderChatCompletionsRequest, -) -> CoreResult { +) -> Result { let body = serde_json::to_vec(&request.body).map_err(|err| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "failed to serialize chat completions request: {err}" )) })?; @@ -32,9 +32,9 @@ pub(super) async fn execute_chat_completions_provider_call( // so the host can still serve it. Everything else here, a timeout // above all, may have reached the provider and been answered. if err.is_connect() || err.is_builder() { - CoreError::Connect(err.to_string()) + Error::Connect(err.to_string()) } else { - CoreError::Network(err.to_string()) + Error::Network(err.to_string()) } })?; @@ -42,17 +42,17 @@ pub(super) async fn execute_chat_completions_provider_call( let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let body: Value = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid chat completions response JSON: {err}")) + Error::InvalidResponse(format!("invalid chat completions response JSON: {err}")) })?; request .config @@ -69,10 +69,10 @@ pub(super) async fn execute_chat_completions_provider_call( /// second kind has already been billed, and a host that keeps a reference /// implementation must not retry those, so collapse them to one variant that /// can only mean the provider was already called. -pub(super) fn as_response_error(err: CoreError) -> CoreError { +pub(super) fn as_response_error(err: Error) -> Error { match err { - already @ (CoreError::InvalidResponse(_) | CoreError::Http { .. }) => already, - other => CoreError::InvalidResponse(other.to_string()), + already @ (Error::InvalidResponse(_) | Error::Http { .. }) => already, + other => Error::InvalidResponse(other.to_string()), } } @@ -80,7 +80,7 @@ pub(super) fn as_response_error(err: CoreError) -> CoreError { pub(super) async fn signed_headers( request: &ProviderChatCompletionsRequest, body: &[u8], -) -> CoreResult> { +) -> Result, Error> { use std::collections::BTreeMap; use std::time::SystemTime; @@ -101,7 +101,7 @@ pub(super) async fn signed_headers( .iter() .any(|(name, _)| is_sigv4_computed_header(name)) { - return Err(CoreError::Unsupported( + return Err(Error::Unsupported( "request forwards a header AWS SigV4 computes", )); } @@ -137,9 +137,9 @@ pub(super) async fn signed_headers( pub(super) async fn signed_headers( request: &ProviderChatCompletionsRequest, _body: &[u8], -) -> CoreResult> { +) -> Result, Error> { match &request.auth { - ChatCompletionsAuth::AwsSigV4 { .. } => Err(CoreError::Unsupported( + ChatCompletionsAuth::AwsSigV4 { .. } => Err(Error::Unsupported( "AWS SigV4 requires the bedrock-auth feature", )), _ => Ok(request.upstream_headers.clone()), diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index f30ac1a24bf..0d009d36d16 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -6,6 +6,7 @@ //! credentials, and it resolves the provider, translates the conversation, //! calls the provider, and returns a typed OpenAI-shaped response. +use crate::Error; mod client; mod common_utils; pub mod conversation; @@ -17,15 +18,13 @@ pub mod types; use serde_json::{Map, Value}; -use crate::error::CoreResult; - use handler::execute_chat_completions_provider_call; use prepare::{parse_messages, prepare_chat_completions_call, resolve_provider_config}; use types::{ChatCompletionsRequest, ChatCompletionsResponse}; pub async fn chat_completions( request: ChatCompletionsRequest<'_>, -) -> CoreResult { +) -> Result { execute_chat_completions_provider_call(prepare_chat_completions_call(request)?).await } diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index 1e1c8d1bafd..142b2f2aaed 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -1,6 +1,6 @@ use serde_json::Value; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::http_utils::has_header; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; @@ -11,7 +11,7 @@ use super::types::{ChatCompletionsRequest, ChatMessage, ProviderChatCompletionsR pub(super) fn resolve_provider_config<'a>( model: &'a str, custom_llm_provider: Option<&'a str>, -) -> CoreResult<(String, &'static dyn ChatCompletionsProviderConfig)> { +) -> Result<(String, &'static dyn ChatCompletionsProviderConfig), Error> { let provider_info = get_custom_llm_provider(model, custom_llm_provider) .or_else(|| { custom_llm_provider.map(|provider| CustomLlmProvider { @@ -20,35 +20,34 @@ pub(super) fn resolve_provider_config<'a>( }) }) .ok_or_else(|| { - CoreError::InvalidProvider( + Error::InvalidProvider( "unable to resolve custom_llm_provider for chat completions request".to_string(), ) })?; let config = chat_completions_provider_config(provider_info.custom_llm_provider) - .ok_or_else(|| CoreError::InvalidProvider(provider_info.custom_llm_provider.to_string()))?; + .ok_or_else(|| Error::InvalidProvider(provider_info.custom_llm_provider.to_string()))?; Ok((provider_info.model.to_string(), config)) } -pub(super) fn parse_messages(messages: Value) -> CoreResult> { - serde_json::from_value(messages).map_err(|err| { - CoreError::InvalidRequest(format!("invalid chat completions messages: {err}")) - }) +pub(super) fn parse_messages(messages: Value) -> Result, Error> { + serde_json::from_value(messages) + .map_err(|err| Error::InvalidRequest(format!("invalid chat completions messages: {err}"))) } pub(super) fn prepare_chat_completions_call( request: ChatCompletionsRequest<'_>, -) -> CoreResult { +) -> Result { let (model, config) = resolve_provider_config(request.model, request.custom_llm_provider)?; let env_lookup = |key: &str| std::env::var(key).ok(); let messages = parse_messages(request.messages)?; if messages.is_empty() { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "chat completions requires at least one message".to_string(), )); } if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) { - return Err(CoreError::Unsupported(reason.0)); + return Err(Error::Unsupported(reason.0)); } let mut headers = string_headers(request.extra_headers)?; diff --git a/litellm-rust/crates/core/src/chat_completions/tests.rs b/litellm-rust/crates/core/src/chat_completions/tests.rs index e2383723cb0..2858d180e27 100644 --- a/litellm-rust/crates/core/src/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/chat_completions/tests.rs @@ -1,6 +1,6 @@ use serde_json::{Map, Value, json}; -use crate::error::CoreError; +use crate::error::Error; use super::prepare::prepare_chat_completions_call; use super::transformation::ChatCompletionsAuth; @@ -29,7 +29,7 @@ fn request<'a>( /// `ProviderChatCompletionsRequest` deliberately has no `Debug` (its headers /// carry resolved credentials), so unwrap the failure case by hand. -fn decline(request: ChatCompletionsRequest<'_>) -> CoreError { +fn decline(request: ChatCompletionsRequest<'_>) -> Error { match prepare_chat_completions_call(request) { Err(error) => error, Ok(prepared) => panic!("expected a decline, prepared a call to {}", prepared.url), @@ -196,7 +196,7 @@ fn declines_an_unsupported_request_before_resolving_credentials() { call.api_key = None; // No api_key is set and no env is consulted: the gate must run first, so the // error is the decline rather than a missing-credential error. - assert_eq!(decline(call), CoreError::Unsupported("streaming")); + assert_eq!(decline(call), Error::Unsupported("streaming")); } #[test] @@ -208,7 +208,7 @@ fn rejects_an_unknown_provider() { json!([{"role": "user", "content": "hi"}]), json!({}), )), - CoreError::InvalidProvider("openai".to_string()) + Error::InvalidProvider("openai".to_string()) ); } @@ -221,7 +221,7 @@ fn rejects_a_model_with_no_resolvable_provider() { json!([{"role": "user", "content": "hi"}]), json!({}), )), - CoreError::InvalidProvider(_) + Error::InvalidProvider(_) )); } @@ -234,7 +234,7 @@ fn rejects_an_empty_or_malformed_message_list() { json!([]), json!({}), )), - CoreError::InvalidRequest("chat completions requires at least one message".to_string()) + Error::InvalidRequest("chat completions requires at least one message".to_string()) ); assert!(matches!( decline(request( @@ -243,7 +243,7 @@ fn rejects_an_empty_or_malformed_message_list() { json!("not a list"), json!({}), )), - CoreError::InvalidRequest(_) + Error::InvalidRequest(_) )); } @@ -258,7 +258,7 @@ fn rejects_non_string_extra_headers() { call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))])); assert_eq!( decline(call), - CoreError::InvalidRequest( + Error::InvalidRequest( "chat completions extra_headers.x-trace must be a string, got number".to_string() ) ); @@ -374,7 +374,7 @@ async fn a_forwarded_header_the_signer_computes_declines_to_python() { .await .expect_err("{forwarded} should decline instead of being signed"); assert!( - matches!(error, CoreError::Unsupported(_)), + matches!(error, Error::Unsupported(_)), "{forwarded} declined as {error:?}, which the host would not fall back on" ); } @@ -727,7 +727,7 @@ mod round_trip { .expect_err("response cannot be normalized"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::InvalidResponse(_)), + matches!(err, Error::InvalidResponse(_)), "expected a post-send error, got {err:?}" ); } @@ -745,7 +745,7 @@ mod round_trip { .expect_err("response cannot be normalized"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::InvalidResponse(_)), + matches!(err, Error::InvalidResponse(_)), "expected a post-send error, got {err:?}" ); } @@ -763,7 +763,7 @@ mod round_trip { .expect_err("upstream rejects"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::Http { status: 429, .. }), + matches!(err, Error::Http { status: 429, .. }), "expected a 429, got {err:?}" ); } @@ -787,7 +787,7 @@ mod round_trip { .await .expect_err("nothing is listening"); assert!( - matches!(err, CoreError::Connect(_)), + matches!(err, Error::Connect(_)), "expected a pre-send connect failure, got {err:?}" ); } @@ -797,24 +797,24 @@ mod round_trip { use crate::chat_completions::handler::as_response_error; for original in [ - CoreError::MissingField("usage"), - CoreError::Unsupported("non-text response content block"), - CoreError::InvalidRequest("whatever".to_string()), - CoreError::Auth("whatever".to_string()), + Error::MissingField("usage"), + Error::Unsupported("non-text response content block"), + Error::InvalidRequest("whatever".to_string()), + Error::Auth("whatever".to_string()), ] { let label = format!("{original:?}"); assert!( - matches!(as_response_error(original), CoreError::InvalidResponse(_)), + matches!(as_response_error(original), Error::InvalidResponse(_)), "{label} must not stay retryable once the provider has answered" ); } // An upstream status is already unambiguous, so it survives intact. assert!(matches!( - as_response_error(CoreError::Http { + as_response_error(Error::Http { status: 500, body: "boom".to_string() }), - CoreError::Http { status: 500, .. } + Error::Http { status: 500, .. } )); } } diff --git a/litellm-rust/crates/core/src/chat_completions/transformation.rs b/litellm-rust/crates/core/src/chat_completions/transformation.rs index a30ce9dc77c..a0868209305 100644 --- a/litellm-rust/crates/core/src/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/chat_completions/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::error::CoreResult; - use super::types::{ ChatCompletionsResponse, ChatMessage, ChatMessageContent, ProviderChatRequestData, ProviderChatResponseData, @@ -39,7 +38,7 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth( &self, @@ -47,7 +46,7 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn default_headers(&self) -> &'static [(&'static str, &'static str)] { &[("content-type", "application/json")] @@ -91,13 +90,13 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_response( &self, model: &str, response: ProviderChatResponseData, - ) -> CoreResult; + ) -> Result; } pub fn unsupported_param( diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index 739532f8cb5..db3fa2ec704 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -1,9 +1,7 @@ -use thiserror::Error; +use thiserror::Error as ThisError; -pub type CoreResult = Result; - -#[derive(Debug, Error, PartialEq, Eq)] -pub enum CoreError { +#[derive(Debug, ThisError, PartialEq, Eq)] +pub enum Error { #[error("expected {expected}, got {actual}")] InvalidType { expected: &'static str, diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index c541f50275b..10661fadf96 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -3,7 +3,7 @@ use serde_json::{Map, Value}; use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; /// Bound an upstream error body before it crosses a host boundary, so provider /// bodies stay data-minimized. @@ -18,7 +18,7 @@ pub fn truncate_error_body(body: &str) -> String { pub fn string_headers( context: &'static str, extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { extra_headers .unwrap_or_default() .into_iter() @@ -27,7 +27,7 @@ pub fn string_headers( .as_str() .map(|value| (key.clone(), value.to_string())) .ok_or_else(|| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "{context} extra_headers.{key} must be a string, got {}", json_type_name(&value) )) @@ -81,7 +81,7 @@ mod tests { let err = string_headers("chat completions", Some(headers)).expect_err("non-string value"); assert_eq!( err, - CoreError::InvalidRequest( + Error::InvalidRequest( "chat completions extra_headers.x-trace must be a string, got number".to_string() ) ); diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index dce4a425ea0..0e18d24e5d8 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -13,4 +13,4 @@ pub mod responses; pub mod router; pub mod routing_utils; -pub use error::{CoreError, CoreResult}; +pub use error::Error; diff --git a/litellm-rust/crates/core/src/messages/common_utils.rs b/litellm-rust/crates/core/src/messages/common_utils.rs index a14dffbc1fe..8dfdb2e361a 100644 --- a/litellm-rust/crates/core/src/messages/common_utils.rs +++ b/litellm-rust/crates/core/src/messages/common_utils.rs @@ -1,9 +1,8 @@ -use serde_json::{Map, Value}; - -use crate::error::CoreResult; +use crate::Error; use crate::http_utils::string_headers as shared_string_headers; use crate::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG; use crate::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG; +use serde_json::{Map, Value}; use super::transformation::AnthropicMessagesProviderConfig; @@ -23,6 +22,6 @@ pub(super) fn messages_provider_config( pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { shared_string_headers(HEADER_CONTEXT, extra_headers) } diff --git a/litellm-rust/crates/core/src/messages/handler.rs b/litellm-rust/crates/core/src/messages/handler.rs index 1c895f66eba..13a65d86131 100644 --- a/litellm-rust/crates/core/src/messages/handler.rs +++ b/litellm-rust/crates/core/src/messages/handler.rs @@ -1,5 +1,5 @@ use crate::constants::ANTHROPIC_MESSAGES_PROVIDER; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use super::client::http_client; use super::common_utils::truncate_error_body; @@ -7,7 +7,7 @@ use super::types::{AnthropicMessagesResponse, ProviderMessagesRequest}; pub(super) async fn execute_messages_provider_call( request: ProviderMessagesRequest, -) -> CoreResult { +) -> Result { let mut request_builder = http_client().post(&request.url).json(&request.body); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -19,32 +19,31 @@ pub(super) async fn execute_messages_provider_call( let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } - let response = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid messages response JSON: {err}")) - })?; + let response = serde_json::from_str(&text) + .map_err(|err| Error::InvalidResponse(format!("invalid messages response JSON: {err}")))?; request.config.transform_response(&request.model, response) } pub(super) async fn execute_messages_provider_stream( request: ProviderMessagesRequest, -) -> CoreResult { +) -> Result { if request.provider != ANTHROPIC_MESSAGES_PROVIDER { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "streaming messages is not supported for this provider".to_string(), )); } @@ -60,14 +59,14 @@ pub(super) async fn execute_messages_provider_stream( let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); if !status.is_success() { let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; - return Err(CoreError::Http { + .map_err(|err| Error::Network(err.to_string()))?; + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index acb36d89daf..ee2877e61fc 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -7,6 +7,7 @@ //! is the streaming variant; it hands the raw upstream response back so a host //! can splice the event stream to its own caller. +use crate::Error; mod client; mod common_utils; mod handler; @@ -14,17 +15,15 @@ mod prepare; pub mod transformation; pub mod types; -use crate::error::CoreResult; - use handler::{execute_messages_provider_call, execute_messages_provider_stream}; use prepare::prepare_messages_call; use types::{AnthropicMessagesResponse, MessagesRequest}; -pub async fn messages(request: MessagesRequest<'_>) -> CoreResult { +pub async fn messages(request: MessagesRequest<'_>) -> Result { execute_messages_provider_call(prepare_messages_call(request)?).await } -pub async fn messages_stream(request: MessagesRequest<'_>) -> CoreResult { +pub async fn messages_stream(request: MessagesRequest<'_>) -> Result { execute_messages_provider_stream(prepare_messages_call(request)?).await } diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index 94b5b1eaed7..3b253ac3766 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}; @@ -7,7 +7,7 @@ use super::types::{MessagesRequest, ProviderMessagesRequest}; pub(super) fn prepare_messages_call( request: MessagesRequest<'_>, -) -> CoreResult { +) -> Result { let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider) .or_else(|| { request @@ -18,7 +18,7 @@ pub(super) fn prepare_messages_call( }) }) .ok_or_else(|| { - CoreError::InvalidProvider( + Error::InvalidProvider( "unable to resolve custom_llm_provider for messages request".to_string(), ) })?; @@ -26,7 +26,7 @@ pub(super) fn prepare_messages_call( let provider = provider_info.custom_llm_provider; let config = messages_provider_config(provider) - .ok_or_else(|| CoreError::InvalidProvider(provider.to_string()))?; + .ok_or_else(|| Error::InvalidProvider(provider.to_string()))?; let env_lookup = |key: &str| std::env::var(key).ok(); let mut headers = string_headers(request.extra_headers)?; @@ -53,11 +53,11 @@ pub(super) fn prepare_messages_call( let url = config.complete_url(request.api_base, &model, &env_lookup)?; let typed_request = serde_json::from_value(request.body).map_err(|err| { - CoreError::InvalidRequest(format!("invalid Anthropic messages request: {err}")) + Error::InvalidRequest(format!("invalid Anthropic messages request: {err}")) })?; let transformed = config.transform_request(typed_request)?; let body = serde_json::to_value(transformed).map_err(|err| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "failed to serialize Anthropic messages request: {err}" )) })?; diff --git a/litellm-rust/crates/core/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs index 9fc1763683b..df9f7051011 100644 --- a/litellm-rust/crates/core/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -4,7 +4,7 @@ use serde_json::{Map, Value, json}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; -use crate::error::CoreError; +use crate::error::Error; use super::common_utils::{ has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body, @@ -77,7 +77,7 @@ fn truncate_error_body_caps_long_payloads() { fn string_headers_rejects_non_string_values() { let headers = json!({"x-count": 3}).as_object().unwrap().clone(); let err = string_headers(Some(headers)).expect_err("non-string header rejected"); - assert!(matches!(err, CoreError::InvalidRequest(_))); + assert!(matches!(err, Error::InvalidRequest(_))); } #[test] @@ -341,7 +341,7 @@ async fn messages_requires_auth_when_no_key_and_no_header() { .await .expect_err("missing auth errors"); - assert!(matches!(err, CoreError::Auth(_))); + assert!(matches!(err, Error::Auth(_))); } #[tokio::test] @@ -420,7 +420,7 @@ async fn messages_maps_provider_error_status_to_http_error() { .await .expect_err("provider error propagates"); - assert!(matches!(err, CoreError::Http { status: 401, .. })); + assert!(matches!(err, Error::Http { status: 401, .. })); } #[tokio::test] @@ -437,5 +437,5 @@ async fn messages_rejects_unsupported_provider() { .await .expect_err("unsupported provider errors"); - assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "openai")); + assert!(matches!(err, Error::InvalidProvider(provider) if provider == "openai")); } diff --git a/litellm-rust/crates/core/src/messages/transformation.rs b/litellm-rust/crates/core/src/messages/transformation.rs index b478e20d24b..673a5728aca 100644 --- a/litellm-rust/crates/core/src/messages/transformation.rs +++ b/litellm-rust/crates/core/src/messages/transformation.rs @@ -1,6 +1,5 @@ -use crate::error::CoreResult; - use super::types::{AnthropicMessagesRequest, AnthropicMessagesResponse}; +use crate::Error; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MessagesAuthStrategy { @@ -23,13 +22,13 @@ pub trait AnthropicMessagesProviderConfig: Sync { api_base: Option<&str>, model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn resolve_api_key( &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth_strategy(&self) -> MessagesAuthStrategy { MessagesAuthStrategy::Header("x-api-key") @@ -49,7 +48,7 @@ pub trait AnthropicMessagesProviderConfig: Sync { fn transform_request( &self, request: AnthropicMessagesRequest, - ) -> CoreResult { + ) -> Result { Ok(request) } @@ -57,7 +56,7 @@ pub trait AnthropicMessagesProviderConfig: Sync { &self, _model: &str, response: AnthropicMessagesResponse, - ) -> CoreResult { + ) -> Result { Ok(response) } } diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs index cb3e735e533..3d3c16c8cb6 100644 --- a/litellm-rust/crates/core/src/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::CoreResult; - use super::types::{OcrRequestData, OcrResponseData}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -43,13 +42,13 @@ pub trait OcrProviderConfig: Sync { model: &str, document: Value, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_ocr_response( &self, model: &str, response_json: Value, - ) -> CoreResult; + ) -> Result; fn complete_url( &self, @@ -57,13 +56,13 @@ pub trait OcrProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn resolve_api_key( &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth_strategy(&self) -> OcrAuthStrategy { OcrAuthStrategy::Bearer diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs index 4534ac0182c..b22de6c47de 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::Error; use serde_json::json; fn messages(value: Value) -> Vec { @@ -19,7 +20,7 @@ fn transform(model: &str, msgs: Value, opts: Value) -> Value { .body } -fn transform_response(body: Value) -> CoreResult { +fn transform_response(body: Value) -> Result { ANTHROPIC_CHAT_COMPLETIONS_CONFIG .transform_response("claude-sonnet-4-5", ProviderChatResponseData { body }) } @@ -390,29 +391,26 @@ fn declines_a_response_carrying_a_non_text_block() { "usage": {"input_tokens": 1, "output_tokens": 1} })) .expect_err("non-text block"); - assert_eq!( - err, - CoreError::Unsupported("non-text response content block") - ); + assert_eq!(err, Error::Unsupported("non-text response content block")); } #[test] fn errors_on_a_response_missing_required_fields() { assert_eq!( transform_response(json!("nope")).expect_err("not an object"), - CoreError::InvalidResponse("messages response is not an object".to_string()) + Error::InvalidResponse("messages response is not an object".to_string()) ); assert_eq!( transform_response(json!({"model": "m", "usage": {}})).expect_err("no content"), - CoreError::MissingField("content") + Error::MissingField("content") ); assert_eq!( transform_response(json!({"model": "m", "content": []})).expect_err("no usage"), - CoreError::MissingField("usage") + Error::MissingField("usage") ); assert_eq!( transform_response(json!({"content": [], "usage": {}})).expect_err("no model"), - CoreError::MissingField("model") + Error::MissingField("model") ); } diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs index 3658642b539..97cc48aa6f2 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs @@ -10,7 +10,7 @@ use crate::chat_completions::types::{ ProviderChatRequestData, ProviderChatResponseData, }; use crate::constants::ANTHROPIC_OAUTH_TOKEN_PREFIX; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::providers::anthropic::messages::transformation::{ complete_anthropic_url, resolve_anthropic_api_key, }; @@ -74,7 +74,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_anthropic_url(api_base, env_lookup)) } @@ -84,7 +84,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(ChatCompletionsAuth::Header { name: "x-api-key", value: resolve_anthropic_api_key(api_key, env_lookup)?, @@ -137,7 +137,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult { + ) -> Result { Ok(ProviderChatRequestData { body: anthropic_body(model, &build_conversation(&messages), optional_params), }) @@ -147,15 +147,16 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { &self, _model: &str, response: ProviderChatResponseData, - ) -> CoreResult { - let body = response.body.as_object().ok_or_else(|| { - CoreError::InvalidResponse("messages response is not an object".into()) - })?; + ) -> Result { + let body = response + .body + .as_object() + .ok_or_else(|| Error::InvalidResponse("messages response is not an object".into()))?; let content = body .get("content") .and_then(Value::as_array) - .ok_or(CoreError::MissingField("content"))?; + .ok_or(Error::MissingField("content"))?; // The route declines tool and thinking requests, so a non-text block // means the response carries something this path never asked for. // Decline rather than silently dropping it; the host falls back. @@ -163,7 +164,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { .iter() .any(|block| block.get("type").and_then(Value::as_str) != Some("text")) { - return Err(CoreError::Unsupported("non-text response content block")); + return Err(Error::Unsupported("non-text response content block")); } let text: String = content .iter() @@ -173,7 +174,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { let usage = body .get("usage") .and_then(Value::as_object) - .ok_or(CoreError::MissingField("usage"))?; + .ok_or(Error::MissingField("usage"))?; let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0); Ok(ChatCompletionsResponse { @@ -181,7 +182,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { model: body .get("model") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("model"))? + .ok_or(Error::MissingField("model"))? .to_string(), choices: vec![ChatCompletionsChoice { index: 0, diff --git a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs index 829f2260d3c..8fcc0f36c7d 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; @@ -17,12 +17,12 @@ pub fn non_empty(value: Option<&str>) -> Option<&str> { pub fn resolve_anthropic_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(ANTHROPIC_API_KEY_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY \ environment variable" .to_string(), @@ -52,7 +52,7 @@ impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { api_base: Option<&str>, _model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_anthropic_url(api_base, env_lookup)) } @@ -60,7 +60,7 @@ impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_anthropic_api_key(api_key, env_lookup) } @@ -121,7 +121,7 @@ mod tests { ); assert!(matches!( resolve_anthropic_api_key(None, &|_| None).expect_err("missing key"), - CoreError::Auth(_) + Error::Auth(_) )); } diff --git a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs index 7b958c77ba3..70dad0300f1 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; use crate::messages::types::{ AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock, @@ -28,12 +28,12 @@ pub const AZURE_ANTHROPIC_MESSAGES_CONFIG: AzureAnthropicMessagesConfig = pub fn resolve_azure_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_KEY_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Azure API Key - Set `api_key` or the AZURE_API_KEY environment variable" .to_string(), ) @@ -43,12 +43,12 @@ pub fn resolve_azure_api_key( pub fn complete_azure_anthropic_url( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let api_base = non_empty(api_base) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_BASE_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Azure API Base - Set `api_base` or the AZURE_API_BASE environment variable. \ Expected format: https://.services.ai.azure.com/anthropic" .to_string(), @@ -147,7 +147,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { api_base: Option<&str>, _model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_azure_anthropic_url(api_base, env_lookup) } @@ -155,7 +155,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_azure_api_key(api_key, env_lookup) } @@ -174,7 +174,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { fn transform_request( &self, request: AnthropicMessagesRequest, - ) -> CoreResult { + ) -> Result { let mut request = fold_system_role_messages(request); if let Some(system) = request.system.as_mut() { strip_scope_from_system(system); @@ -190,7 +190,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { &self, model: &str, response: AnthropicMessagesResponse, - ) -> CoreResult { + ) -> Result { self.anthropic.transform_response(model, response) } } @@ -268,7 +268,7 @@ mod tests { "https://env.services.ai.azure.com/anthropic/v1/messages" ); let err = complete_azure_anthropic_url(Some(" "), &|_| None).expect_err("missing base"); - assert!(matches!(err, CoreError::Auth(_))); + assert!(matches!(err, Error::Auth(_))); } #[test] @@ -284,7 +284,7 @@ mod tests { ); assert!(matches!( resolve_azure_api_key(None, &|_| None).expect_err("missing key"), - CoreError::Auth(_) + Error::Auth(_) )); } diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs index eabd15677cc..b26a7925e8a 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs @@ -1,6 +1,6 @@ use std::collections::BTreeSet; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::{OcrAuthStrategy, OcrProviderConfig, OcrResponseHandling}; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value, json}; @@ -32,17 +32,17 @@ fn resolve_value( env_name: &str, env_lookup: &dyn Fn(&str) -> Option, missing_message: &str, -) -> CoreResult { +) -> Result { non_empty(explicit) .map(str::to_string) .or_else(|| env_lookup(env_name).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| CoreError::Auth(missing_message.to_string())) + .ok_or_else(|| Error::Auth(missing_message.to_string())) } pub fn resolve_azure_ai_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_key, AZURE_AI_API_KEY_ENV, @@ -54,7 +54,7 @@ pub fn resolve_azure_ai_api_key( pub fn resolve_azure_ai_api_base( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_base, AZURE_AI_API_BASE_ENV, @@ -66,7 +66,7 @@ pub fn resolve_azure_ai_api_base( pub fn complete_azure_ai_url( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let base = resolve_azure_ai_api_base(api_base, env_lookup)?; Ok(format!( "{}/providers/mistral/azure/ocr", @@ -77,7 +77,7 @@ pub fn complete_azure_ai_url( pub fn resolve_document_intelligence_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_key, AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV, @@ -89,7 +89,7 @@ pub fn resolve_document_intelligence_api_key( pub fn resolve_document_intelligence_endpoint( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_base, AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV, @@ -127,7 +127,7 @@ fn pages_token_is_valid(token: &str) -> bool { } } -fn normalize_pages_param(pages: &Value) -> CoreResult> { +fn normalize_pages_param(pages: &Value) -> Result, Error> { match pages { Value::String(value) => { let normalized = value @@ -138,7 +138,7 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { if normalized.split(',').all(pages_token_is_valid) { Ok(Some(normalized)) } else { - Err(CoreError::InvalidRequest(format!( + Err(Error::InvalidRequest(format!( "Invalid `pages` string for Azure Document Intelligence: {value:?}. Expected format like '1-3,5,7-9'." ))) } @@ -152,7 +152,7 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { for value in values { let page = value.as_i64().expect("checked is_i64"); if page < 0 { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "`pages` integers must be >= 0 (Mistral 0-based indices)".to_string(), )); } @@ -176,16 +176,16 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { if normalized.split(',').all(pages_token_is_valid) { return Ok(Some(normalized)); } - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Invalid `pages` list for Azure Document Intelligence: {values:?}. Expected tokens like '1' or '3-5'." ))); } - Err(CoreError::InvalidRequest( + Err(Error::InvalidRequest( "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." .to_string(), )) } - _ => Err(CoreError::InvalidRequest( + _ => Err(Error::InvalidRequest( "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." .to_string(), )), @@ -197,7 +197,7 @@ pub fn complete_document_intelligence_url( model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let endpoint = resolve_document_intelligence_endpoint(api_base, env_lookup)?; let mut url = format!( "{}/documentintelligence/documentModels/{}:analyze?api-version={}", @@ -216,20 +216,20 @@ pub fn complete_document_intelligence_url( Ok(url) } -fn document_url_from_mistral_document(document: &Value) -> CoreResult<&str> { - let object = document.as_object().ok_or_else(|| CoreError::InvalidType { +fn document_url_from_mistral_document(document: &Value) -> Result<&str, Error> { + let object = document.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(document), })?; let doc_type = object .get("type") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("document.type"))?; + .ok_or(Error::MissingField("document.type"))?; let field_name = match doc_type { "document_url" => "document_url", "image_url" => "image_url", other => { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Invalid document type: {other}. Must be 'document_url' or 'image_url'" ))); } @@ -238,7 +238,7 @@ fn document_url_from_mistral_document(document: &Value) -> CoreResult<&str> { .get(field_name) .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField(field_name)) + .ok_or(Error::MissingField(field_name)) } fn extract_base64_from_data_uri(data_uri: &str) -> &str { @@ -290,7 +290,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } @@ -298,7 +298,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -308,7 +308,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_azure_ai_url(api_base, env_lookup) } @@ -316,7 +316,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_azure_ai_api_key(api_key, env_lookup) } @@ -335,7 +335,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { _model: &str, document: Value, _optional_params: Map, - ) -> CoreResult { + ) -> Result { let document_url = document_url_from_mistral_document(&document)?; let mut data = Map::new(); if document_url.starts_with("data:") { @@ -359,19 +359,19 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; let status = response .get("status") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("status"))?; + .ok_or(Error::MissingField("status"))?; if status != "succeeded" { - return Err(CoreError::InvalidResponse(format!( + return Err(Error::InvalidResponse(format!( "Azure Document Intelligence analysis failed with status: {status}" ))); } @@ -414,7 +414,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_document_intelligence_url(api_base, model, optional_params, env_lookup) } @@ -422,7 +422,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_document_intelligence_api_key(api_key, env_lookup) } diff --git a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs index 5e885734182..bb4f6afe5f9 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs @@ -6,7 +6,7 @@ use crate::audio_transcription::transformation::{ use crate::audio_transcription::types::{ AudioTranscriptionRequestData, AudioTranscriptionResponseData, }; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; pub use super::aws_base::{aws_auth_config, bedrock_model_id_and_region, resolve_bedrock_region}; use super::constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}; @@ -18,8 +18,8 @@ pub static BEDROCK_AUDIO_TRANSCRIPTION_CONFIG: BedrockAudioTranscriptionConfig = pub struct BedrockAudioTranscriptionConfig; -fn audio_fields(audio: Value) -> CoreResult<(String, String)> { - let object = audio.as_object().ok_or_else(|| CoreError::InvalidType { +fn audio_fields(audio: Value) -> Result<(String, String), Error> { + let object = audio.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&audio), })?; @@ -27,13 +27,13 @@ fn audio_fields(audio: Value) -> CoreResult<(String, String)> { .get("data") .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField("audio.data"))?; + .ok_or(Error::MissingField("audio.data"))?; let format = object .get("format") .and_then(Value::as_str) .filter(|value| matches!(*value, "wav" | "mp3" | "flac" | "ogg")) .ok_or_else(|| { - CoreError::InvalidRequest("audio.format must be wav, mp3, flac, or ogg".to_string()) + Error::InvalidRequest("audio.format must be wav, mp3, flac, or ogg".to_string()) })?; Ok((data.to_string(), format.to_string())) } @@ -55,7 +55,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { _model: &str, audio: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { let (data, format) = audio_fields(audio)?; let mut instruction = "Transcribe the audio. Respond with only the transcript.".to_string(); if let Some(language) = optional_string(&optional_params, "language") { @@ -87,14 +87,14 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { &self, _model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let content = response_json .get("output") .and_then(|value| value.get("message")) .and_then(|value| value.get("content")) .and_then(Value::as_array) .ok_or_else(|| { - CoreError::InvalidResponse("Bedrock response has no output content".to_string()) + Error::InvalidResponse("Bedrock response has no output content".to_string()) })?; let mut text = String::new(); for block in content { @@ -111,7 +111,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (model_id, model_region) = bedrock_model_id_and_region(model); let region = resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup); let endpoint = optional_params @@ -133,7 +133,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (_, model_region) = bedrock_model_id_and_region(model); Ok(AudioTranscriptionAuth::AwsSigV4 { region: resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup), diff --git a/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs b/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs index b11639aa09b..e5e52bfce95 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs @@ -4,7 +4,7 @@ use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use crate::caching::in_memory_cache::InMemoryCache; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use aws_credential_types::Credentials; use aws_credential_types::provider::ProvideCredentials; use aws_sigv4::http_request::{ @@ -197,7 +197,7 @@ pub fn classify_auth( pub async fn resolve_credentials( config: AwsAuthConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), -) -> CoreResult { +) -> Result { let resolved = config.clone().with_environment(env_lookup); let flow = classify_auth(config, env_lookup); match flow { @@ -244,9 +244,10 @@ pub async fn resolve_credentials( let provider = aws_config::profile::ProfileFileCredentialsProvider::builder() .profile_name(name) .build(); - provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS profile credentials failed: {error}")) - }) + provider + .provide_credentials() + .await + .map_err(|error| Error::Auth(format!("AWS profile credentials failed: {error}"))) } AwsAuthFlow::AssumeRole { role, session_name } => { if is_already_running_as_role(&role, &resolved).await? { @@ -260,7 +261,7 @@ pub async fn resolve_credentials( .build() .await; let credentials = provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS default credentials failed: {error}")) + Error::Auth(format!("AWS default credentials failed: {error}")) })?; set_cached_credentials( key, @@ -301,7 +302,7 @@ pub async fn resolve_credentials( provider .provide_credentials() .await - .map_err(|error| CoreError::Auth(format!("AWS role credentials failed: {error}"))) + .map_err(|error| Error::Auth(format!("AWS role credentials failed: {error}"))) } AwsAuthFlow::WebIdentity { token, @@ -325,13 +326,13 @@ pub async fn resolve_credentials( .send() .await .map_err(|error| { - CoreError::Auth(format!("AWS web identity credentials failed: {error}")) + Error::Auth(format!("AWS web identity credentials failed: {error}")) })?; let credentials = response.credentials().ok_or_else(|| { - CoreError::Auth("AWS web identity response had no credentials".to_string()) + Error::Auth("AWS web identity response had no credentials".to_string()) })?; let expiration = SystemTime::try_from(*credentials.expiration()).map_err(|error| { - CoreError::Auth(format!("AWS web identity expiration was invalid: {error}")) + Error::Auth(format!("AWS web identity expiration was invalid: {error}")) })?; Ok(Credentials::new( credentials.access_key_id(), @@ -350,9 +351,10 @@ pub async fn resolve_credentials( aws_config::default_provider::credentials::DefaultCredentialsChain::builder() .build() .await; - let credentials = provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS default credentials failed: {error}")) - })?; + let credentials = provider + .provide_credentials() + .await + .map_err(|error| Error::Auth(format!("AWS default credentials failed: {error}")))?; set_cached_credentials( key, credentials.clone(), @@ -363,7 +365,7 @@ pub async fn resolve_credentials( } } -async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> CoreResult { +async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> Result { if role_identity(role).is_none() { return Ok(false); } @@ -437,7 +439,7 @@ pub fn sign_bedrock_post( region: &str, credentials: &Credentials, signing_time: SystemTime, -) -> CoreResult> { +) -> Result, Error> { let identity: Identity = credentials.clone().into(); let params = v4::SigningParams::builder() .identity(&identity) @@ -447,14 +449,14 @@ pub fn sign_bedrock_post( .settings(SigningSettings::default()) .build() .map(SigningParams::from) - .map_err(|error| CoreError::Auth(format!("AWS signing parameters failed: {error}")))?; + .map_err(|error| Error::Auth(format!("AWS signing parameters failed: {error}")))?; let header_refs = headers .iter() .map(|(name, value)| (name.as_str(), value.as_str())); let request = SignableRequest::new("POST", url, header_refs, SignableBody::Bytes(body)) - .map_err(|error| CoreError::Auth(format!("AWS signable request failed: {error}")))?; + .map_err(|error| Error::Auth(format!("AWS signable request failed: {error}")))?; let (instructions, _) = sign(request, ¶ms) - .map_err(|error| CoreError::Auth(format!("AWS request signing failed: {error}")))? + .map_err(|error| Error::Auth(format!("AWS request signing failed: {error}")))? .into_parts(); Ok(instructions .headers() diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs index 4b75dcb8e9d..c86f061b9ca 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::Error; use serde_json::json; fn messages(value: Value) -> Vec { @@ -23,7 +24,7 @@ fn transform(msgs: Value, opts: Value) -> Value { .body } -fn transform_response(body: Value) -> CoreResult { +fn transform_response(body: Value) -> Result { BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response( "anthropic.claude-sonnet-4-5-v1:0", ProviderChatResponseData { body }, @@ -478,25 +479,22 @@ fn declines_a_response_carrying_a_tool_use_block() { "usage": {"inputTokens": 1, "outputTokens": 1} })) .expect_err("tool use block"); - assert_eq!( - err, - CoreError::Unsupported("non-text response content block") - ); + assert_eq!(err, Error::Unsupported("non-text response content block")); } #[test] fn errors_on_a_response_missing_required_fields() { assert_eq!( transform_response(json!("nope")).expect_err("not an object"), - CoreError::InvalidResponse("converse response is not an object".to_string()) + Error::InvalidResponse("converse response is not an object".to_string()) ); assert_eq!( transform_response(json!({"usage": {}})).expect_err("no output"), - CoreError::MissingField("output.message.content") + Error::MissingField("output.message.content") ); assert_eq!( transform_response(json!({"output": {"message": {"content": []}}})).expect_err("no usage"), - CoreError::MissingField("usage") + Error::MissingField("usage") ); } diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs index b107950748e..ef5f44b4a14 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs @@ -11,7 +11,7 @@ use crate::chat_completions::types::{ ChatCompletionsUsage, ChatMessage, ChatMessageContent, ProviderChatRequestData, ProviderChatResponseData, }; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use super::super::aws_base::{bedrock_model_id_and_region, resolve_bedrock_region}; use super::super::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE}; @@ -110,7 +110,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (model_id, model_region) = bedrock_model_id_and_region(model); let region = resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup); let endpoint = optional_params @@ -137,7 +137,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { // Python reads `api_key` as the Bedrock bearer token and consults the // env only when the caller passed none, so a caller-supplied empty key // falls through to SigV4 without reaching for the environment. An @@ -208,7 +208,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { _model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult { + ) -> Result { Ok(ProviderChatRequestData { body: converse_body(&build_conversation(&messages), &optional_params), }) @@ -218,17 +218,18 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { &self, model: &str, response: ProviderChatResponseData, - ) -> CoreResult { - let body = response.body.as_object().ok_or_else(|| { - CoreError::InvalidResponse("converse response is not an object".into()) - })?; + ) -> Result { + let body = response + .body + .as_object() + .ok_or_else(|| Error::InvalidResponse("converse response is not an object".into()))?; let content = body .get("output") .and_then(|output| output.get("message")) .and_then(|message| message.get("content")) .and_then(Value::as_array) - .ok_or(CoreError::MissingField("output.message.content"))?; + .ok_or(Error::MissingField("output.message.content"))?; // The route declines tool requests, so anything other than a text block // is something this path never asked for. Decline; the host falls back. if content.iter().any(|block| { @@ -236,7 +237,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { .as_object() .is_none_or(|block| block.len() != 1 || !block.contains_key("text")) }) { - return Err(CoreError::Unsupported("non-text response content block")); + return Err(Error::Unsupported("non-text response content block")); } let text: String = content .iter() @@ -246,7 +247,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { let usage = body .get("usage") .and_then(Value::as_object) - .ok_or(CoreError::MissingField("usage"))?; + .ok_or(Error::MissingField("usage"))?; let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0); let computed = usage_from_parts( field("inputTokens"), diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs index dc720cc4244..6a8a38204a9 100644 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value}; @@ -47,7 +47,7 @@ pub fn complete_url(api_base: Option<&str>) -> String { /// Resolve the Mistral API key from the explicit param or the environment. /// -/// Blank/whitespace values are treated as absent. Returns `CoreError::Auth` +/// Blank/whitespace values are treated as absent. Returns `Error::Auth` /// when no usable key is available. /// /// Note: the env fallback only reads the process environment. Secret-manager @@ -56,13 +56,13 @@ pub fn complete_url(api_base: Option<&str>) -> String { pub fn resolve_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) .map(str::to_string) .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty())) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } pub struct MistralOcrConfig; @@ -79,9 +79,9 @@ impl OcrProviderConfig for MistralOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { if !document.is_object() { - return Err(CoreError::InvalidType { + return Err(Error::InvalidType { expected: "object", actual: json_type_name(&document), }); @@ -104,10 +104,10 @@ impl OcrProviderConfig for MistralOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response_object = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; @@ -140,7 +140,7 @@ impl OcrProviderConfig for MistralOcrConfig { _model: &str, _optional_params: &Map, _env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_url(api_base)) } @@ -148,7 +148,7 @@ impl OcrProviderConfig for MistralOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_api_key(api_key, env_lookup) } } @@ -165,11 +165,11 @@ pub fn transform_ocr_request( model: &str, document: Value, optional_params: Map, -) -> CoreResult { +) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } -pub fn transform_ocr_response(model: &str, response_json: Value) -> CoreResult { +pub fn transform_ocr_response(model: &str, response_json: Value) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -250,7 +250,7 @@ mod tests { assert_eq!( err, - CoreError::InvalidType { + Error::InvalidType { expected: "object", actual: "string", } @@ -307,6 +307,6 @@ mod tests { #[test] fn resolve_api_key_errors_when_absent() { let err = resolve_api_key(None, &|_| None).expect_err("missing key should error"); - assert_eq!(err, CoreError::Auth(MISSING_KEY_MESSAGE.to_string())); + assert_eq!(err, Error::Auth(MISSING_KEY_MESSAGE.to_string())); } } diff --git a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs index b3f6b03b28a..f1985f81b7d 100644 --- a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs +++ b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::realtime::transformation::RealtimeProviderConfig; use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; @@ -72,7 +72,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { &self, event: &RealtimeEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(RealtimeTransformResult::passthrough(event.clone())) } @@ -80,7 +80,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { &self, event: &RealtimeEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(RealtimeTransformResult::passthrough(event.clone())) } } @@ -88,14 +88,14 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { pub fn transform_realtime_request( event: &RealtimeEvent, model: &str, -) -> CoreResult { +) -> Result { OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model) } pub fn transform_realtime_response( event: &RealtimeEvent, model: &str, -) -> CoreResult { +) -> Result { OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model) } diff --git a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs index e15197c468c..be86bb90311 100644 --- a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs +++ b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::responses::types::{ResponsesWsEvent, ResponsesWsTransformResult}; use crate::responses::websocket::{ResponsesWebSocketProviderConfig, enforce_model}; @@ -15,7 +15,7 @@ impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig { &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult { + ) -> Result { Ok(ResponsesWsTransformResult::passthrough(enforce_model( event, model, ))) @@ -25,7 +25,7 @@ impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig { &self, event: &ResponsesWsEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(ResponsesWsTransformResult::passthrough(event.clone())) } } diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs index 6300149c237..ee095447028 100644 --- a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value, json}; @@ -43,7 +43,7 @@ pub fn is_deepseek_model(model: &str) -> bool { pub fn resolve_vertex_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) @@ -51,7 +51,7 @@ pub fn resolve_vertex_api_key( .or_else(|| env_lookup(VERTEX_AI_API_KEY_ENV).filter(|key| !key.trim().is_empty())) .or_else(|| env_lookup(VERTEXAI_API_KEY_ENV).filter(|key| !key.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Vertex AI access token - pass api_key or provide Authorization via extra_headers" .to_string(), ) @@ -61,12 +61,12 @@ pub fn resolve_vertex_api_key( fn vertex_project( params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { string_param(params, &["vertex_project", "vertex_ai_project"]) .map(str::to_string) .or_else(|| env_lookup(VERTEXAI_PROJECT_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::InvalidRequest( + Error::InvalidRequest( "Missing vertex_project - Set VERTEXAI_PROJECT environment variable or pass vertex_project parameter" .to_string(), ) @@ -99,7 +99,7 @@ pub fn complete_vertex_mistral_url( model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let project = vertex_project(optional_params, env_lookup)?; let location = vertex_location(optional_params, env_lookup); let base = vertex_mistral_api_base(api_base, &location); @@ -112,7 +112,7 @@ pub fn complete_vertex_deepseek_url( api_base: Option<&str>, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let project = vertex_project(optional_params, env_lookup)?; let location = vertex_location(optional_params, env_lookup); let base = api_base @@ -125,20 +125,20 @@ pub fn complete_vertex_deepseek_url( )) } -fn document_content_item(document: &Value) -> CoreResult { - let object = document.as_object().ok_or_else(|| CoreError::InvalidType { +fn document_content_item(document: &Value) -> Result { + let object = document.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(document), })?; let doc_type = object .get("type") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("document.type"))?; + .ok_or(Error::MissingField("document.type"))?; let url_field = match doc_type { "image_url" => "image_url", "document_url" => "document_url", other => { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Unsupported document type: {other}. Expected 'image_url' or 'document_url'" ))); } @@ -147,7 +147,7 @@ fn document_content_item(document: &Value) -> CoreResult { .get(url_field) .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField(url_field))?; + .ok_or(Error::MissingField(url_field))?; Ok(json!({ "type": "image_url", @@ -163,7 +163,7 @@ fn deepseek_model_name(model: &str) -> String { } } -fn first_choice_content(response: &Value) -> CoreResult { +fn first_choice_content(response: &Value) -> Result { response .get("choices") .and_then(Value::as_array) @@ -176,9 +176,7 @@ fn first_choice_content(response: &Value) -> CoreResult { Value::Object(_) => true, _ => false, }) - .ok_or_else(|| { - CoreError::InvalidResponse("No content in DeepSeek OCR response".to_string()) - }) + .ok_or_else(|| Error::InvalidResponse("No content in DeepSeek OCR response".to_string())) } fn ocr_data_from_content(content: Value, usage: Option, model: &str) -> Value { @@ -219,7 +217,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } @@ -227,7 +225,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -237,7 +235,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_vertex_mistral_url(api_base, model, optional_params, env_lookup) } @@ -245,7 +243,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_vertex_api_key(api_key, env_lookup) } @@ -264,7 +262,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { let mut data = Map::new(); data.insert( "model".to_string(), @@ -289,10 +287,10 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; @@ -314,7 +312,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { }); } - let object = ocr_data.as_object().ok_or_else(|| CoreError::InvalidType { + let object = ocr_data.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&ocr_data), })?; @@ -346,7 +344,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { _model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_vertex_deepseek_url(api_base, optional_params, env_lookup) } @@ -354,7 +352,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_vertex_api_key(api_key, env_lookup) } } diff --git a/litellm-rust/crates/core/src/realtime/transformation.rs b/litellm-rust/crates/core/src/realtime/transformation.rs index 69b88687000..b08084514ef 100644 --- a/litellm-rust/crates/core/src/realtime/transformation.rs +++ b/litellm-rust/crates/core/src/realtime/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; pub trait RealtimeProviderConfig { @@ -11,12 +11,12 @@ pub trait RealtimeProviderConfig { &self, event: &RealtimeEvent, model: &str, - ) -> CoreResult; + ) -> Result; /// Transform a backend → client event before it is forwarded downstream. fn transform_realtime_response( &self, event: &RealtimeEvent, model: &str, - ) -> CoreResult; + ) -> Result; } diff --git a/litellm-rust/crates/core/src/responses/instrumentation.rs b/litellm-rust/crates/core/src/responses/instrumentation.rs index ec04571da14..b1098f4d386 100644 --- a/litellm-rust/crates/core/src/responses/instrumentation.rs +++ b/litellm-rust/crates/core/src/responses/instrumentation.rs @@ -5,9 +5,9 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde_json::Value; +use crate::Error; use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType}; -use crate::{CoreError, CoreResult}; #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ResponsesWsUsage { @@ -205,7 +205,7 @@ impl ResponsesWsInstrumentation { } } -type LifecycleFuture<'a, T> = Pin> + Send + 'a>>; +type LifecycleFuture<'a, T> = Pin> + Send + 'a>>; impl CallLifecycleHooks<(), (), ()> for ResponsesWsInstrumentation { type PreCallFuture<'a> = LifecycleFuture<'a, ()>; @@ -246,7 +246,7 @@ impl CallLifecycleHooks<(), (), ()> for ResponsesWsInstrumentation { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -342,7 +342,7 @@ mod tests { ), (), &instrumentation, - |_| async { Ok::<(), CoreError>(()) }, + |_| async { Ok::<(), Error>(()) }, ) .await; diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/core/src/responses/websocket.rs index 92dc19627a0..5d037e9cf1b 100644 --- a/litellm-rust/crates/core/src/responses/websocket.rs +++ b/litellm-rust/crates/core/src/responses/websocket.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}; use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult}; @@ -19,13 +19,13 @@ pub trait ResponsesWebSocketProviderConfig: Sync { &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult; + ) -> Result; fn transform_ws_response( &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult; + ) -> Result; } pub fn complete_websocket_url( diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 746e0770f9b..68aa9436b15 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -10,7 +10,7 @@ use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompleti use litellm_core::chat_completions::{ chat_completions as run_chat_completions, chat_completions_decline_reason, }; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::messages::messages as run_messages; use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; use litellm_python_interop::{from_py, release_count, release_gil, to_py}; @@ -54,13 +54,13 @@ fn chat_completions_response_to_py( to_py(py, &response) } -fn core_error_to_pyerr(err: CoreError) -> PyErr { +fn core_error_to_pyerr(err: Error) -> PyErr { match err { - CoreError::Auth(message) => PyValueError::new_err(message), - CoreError::InvalidProvider(_) - | CoreError::InvalidRequest(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) => PyValueError::new_err(err.to_string()), + Error::Auth(message) => PyValueError::new_err(message), + Error::InvalidProvider(_) + | Error::InvalidRequest(_) + | Error::InvalidType { .. } + | Error::MissingField(_) => PyValueError::new_err(err.to_string()), other => PyRuntimeError::new_err(other.to_string()), } } @@ -71,22 +71,22 @@ fn core_error_to_pyerr(err: CoreError) -> PyErr { /// Everything raised before the request goes out is safe for the host to retry /// on its own path; anything after it is not, because the provider has already /// done the work and billed for it. -fn chat_completions_error_to_pyerr(err: CoreError) -> PyErr { +fn chat_completions_error_to_pyerr(err: Error) -> PyErr { match err { - CoreError::Unsupported(_) - | CoreError::Auth(_) - | CoreError::InvalidProvider(_) - | CoreError::InvalidRequest(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) - | CoreError::Routing(_) + Error::Unsupported(_) + | Error::Auth(_) + | Error::InvalidProvider(_) + | Error::InvalidRequest(_) + | Error::InvalidType { .. } + | Error::MissingField(_) + | Error::Routing(_) // Nothing reached the provider, so serving it on Python cannot double // bill and is the only way the caller gets an answer at all. - | CoreError::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), - CoreError::Http { status, body } => { + | Error::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), + Error::Http { status, body } => { RustUpstreamError::new_err((status, format!("{status}: {body}"))) } - CoreError::Network(message) | CoreError::InvalidResponse(message) => { + Error::Network(message) | Error::InvalidResponse(message) => { RustUpstreamError::new_err((0u16, message)) } } From df421b42dba812a48ac6bf880ea8d0986e53399c Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 11:49:42 -0700 Subject: [PATCH 53/67] refactor(rust): move audio transcription into core --- .../src/audio_transcription/common_utils.rs | 46 ---------- .../src/audio_transcription/handler.rs | 84 ----------------- .../src/audio_transcription/hooks.rs | 82 +++++++---------- .../ai-gateway/src/audio_transcription/mod.rs | 4 +- .../src/audio_transcription/types.rs | 11 --- .../core/src/audio_transcription/client.rs | 14 +++ .../core/src/audio_transcription/handler.rs | 91 +++++++++++++++++++ .../core/src/audio_transcription/mod.rs | 18 ++++ .../core/src/audio_transcription/prepare.rs | 72 +++++++++++++++ .../core/src/audio_transcription/tests.rs | 50 ++++++++++ .../core/src/audio_transcription/types.rs | 53 ++++++++++- litellm-rust/crates/core/src/constants.rs | 2 + litellm-rust/crates/python-bridge/src/lib.rs | 14 +-- 13 files changed, 336 insertions(+), 205 deletions(-) delete mode 100644 litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs create mode 100644 litellm-rust/crates/core/src/audio_transcription/client.rs create mode 100644 litellm-rust/crates/core/src/audio_transcription/handler.rs create mode 100644 litellm-rust/crates/core/src/audio_transcription/prepare.rs create mode 100644 litellm-rust/crates/core/src/audio_transcription/tests.rs diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs deleted file mode 100644 index 140bc8aeea8..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs +++ /dev/null @@ -1,46 +0,0 @@ -use litellm_core::audio_transcription::transformation::AudioTranscriptionProviderConfig; -use litellm_core::error::Error; -use litellm_core::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG; -use serde_json::{Map, Value}; -use std::collections::BTreeMap; - -pub(super) fn audio_transcription_provider_config( - provider: &str, -) -> Option<&'static dyn AudioTranscriptionProviderConfig> { - match provider { - "bedrock" => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG), - _ => None, - } -} - -pub(super) fn string_headers( - headers: Option>, -) -> Result, Error> { - headers - .unwrap_or_default() - .into_iter() - .map(|(key, value)| { - value - .as_str() - .map(|value| (key.clone(), value.to_string())) - .ok_or_else(|| { - Error::InvalidRequest(format!( - "audio transcription extra_headers.{key} must be a string" - )) - }) - }) - .collect() -} - -pub(super) fn has_header(headers: &BTreeMap, name: &str) -> bool { - headers.keys().any(|key| key.eq_ignore_ascii_case(name)) -} - -pub(super) fn truncate_error_body(body: &str) -> String { - let truncated: String = body.chars().take(256).collect(); - if truncated.chars().count() == body.chars().count() { - truncated - } else { - format!("{truncated}... (truncated)") - } -} diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs deleted file mode 100644 index 1bdd4ae72a2..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs +++ /dev/null @@ -1,84 +0,0 @@ -use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth; -use litellm_core::error::Error; -use litellm_core::providers::bedrock::audio_transcription::aws_auth_config; -use litellm_core::providers::bedrock::aws_base::{resolve_credentials, sign_bedrock_post}; -use serde_json::Value; -use std::time::SystemTime; - -use super::common_utils::truncate_error_body; -use super::types::ProviderAudioTranscriptionRequest; -use crate::client::http_client; - -pub(crate) async fn execute_audio_transcription_provider_call( - request: ProviderAudioTranscriptionRequest, -) -> Result { - let body = serde_json::to_vec(&request.body) - .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?; - let mut request_builder = http_client().post(&request.url).body(body.clone()); - for (key, value) in &request.upstream_headers { - request_builder = request_builder.header(key, value); - } - if let Some(duration) = request.timeout { - request_builder = request_builder.timeout(duration); - } - let response = request_builder - .send() - .await - .map_err(|error| Error::Network(error.to_string()))?; - let status = response.status(); - let text = response - .text() - .await - .map_err(|error| Error::Network(error.to_string()))?; - if !status.is_success() { - return Err(Error::Http { - status: status.as_u16(), - body: truncate_error_body(&text), - }); - } - let response_json: Value = serde_json::from_str(&text) - .map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?; - Ok(request - .config - .transform_transcription_response(&request.model, response_json)? - .into_json()) -} - -pub(crate) async fn sign_request( - request: &ProviderAudioTranscriptionRequest, - optional_params: &serde_json::Map, -) -> Result { - let env_lookup = environment_lookup; - let auth = request - .config - .auth_strategy(&request.model, optional_params, &env_lookup)?; - let body = serde_json::to_vec(&request.body) - .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?; - let mut headers = super::common_utils::string_headers(None)?; - headers.insert("Content-Type".to_string(), "application/json".to_string()); - headers.extend(request.upstream_headers.iter().cloned()); - match auth { - AudioTranscriptionAuth::Bearer => {} - AudioTranscriptionAuth::AwsSigV4 { region, .. } => { - let credentials = - resolve_credentials(aws_auth_config(optional_params, &env_lookup), &env_lookup) - .await?; - headers.extend(sign_bedrock_post( - &request.url, - &body, - &headers, - ®ion, - &credentials, - SystemTime::now(), - )?); - } - } - Ok(ProviderAudioTranscriptionRequest { - upstream_headers: headers.into_iter().collect(), - ..request.clone() - }) -} - -pub(super) fn environment_lookup(key: &str) -> Option { - std::env::var(key).ok() -} diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs index 5e1240de759..dbe2d3a325b 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs @@ -1,13 +1,14 @@ -use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth; +use litellm_core::audio_transcription::{ + AudioTranscriptionRequest as CoreAudioTranscriptionRequest, ProviderAudioTranscriptionRequest, + prepare_audio_transcription_provider_call, +}; use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; use litellm_core::error::Error; use serde_json::{Map, Value, json}; use std::future::Future; use std::pin::Pin; -use super::common_utils::{audio_transcription_provider_config, has_header, string_headers}; -use super::handler::sign_request; -use super::types::{PreparedAudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; +use super::types::PreparedAudioTranscriptionRequest; use crate::integrations::custom_guardrail::{ CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest, }; @@ -88,46 +89,29 @@ impl AudioTranscriptionLifecycleHooks { &self, request: PreparedAudioTranscriptionRequest, ) -> Result { - let config = audio_transcription_provider_config(&request.custom_llm_provider) - .ok_or_else(|| Error::InvalidProvider(request.custom_llm_provider.clone()))?; - let env_lookup = super::handler::environment_lookup; - let headers = string_headers(request.extra_headers)?; - let url = config.complete_url( - request.api_base.as_deref(), - &request.model, - &request.optional_params, - &env_lookup, - )?; - let filtered_params = config.map_transcription_params(&request.optional_params); - let body = config.transform_transcription_request( - &request.model, - request.audio, - filtered_params, - )?; - let auth = config.auth_strategy(&request.model, &request.optional_params, &env_lookup)?; - let mut upstream_headers = headers.into_iter().collect::>(); - if matches!(auth, AudioTranscriptionAuth::Bearer) - && !has_header( - &upstream_headers - .iter() - .cloned() - .collect::>(), - "authorization", - ) - && let Some(api_key) = request.api_key.as_deref() - { - upstream_headers.push(("Authorization".to_string(), format!("Bearer {api_key}"))); - } - let provider_request = ProviderAudioTranscriptionRequest { - model: request.model, - config, - url, - body: body.body, - upstream_headers, - timeout: request.timeout, - }; - let provider_request = self.run_during_call_guardrails(provider_request).await?; - sign_request(&provider_request, &request.optional_params).await + let PreparedAudioTranscriptionRequest { + model, + custom_llm_provider, + audio, + api_key, + api_base, + extra_headers, + optional_params, + timeout, + .. + } = request; + let provider_request = + prepare_audio_transcription_provider_call(CoreAudioTranscriptionRequest { + model: &model, + audio, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: Some(&custom_llm_provider), + extra_headers, + optional_params, + timeout, + })?; + self.run_during_call_guardrails(provider_request).await } async fn run_during_call_guardrails( @@ -142,10 +126,10 @@ impl AudioTranscriptionLifecycleHooks { .run_during_call( &guardrail_context(&self.request_metadata), GuardrailRequest::new(json!({ - "model": request.model, - "custom_llm_provider": "bedrock", - "url": request.url, - "body": request.body, + "model": request.model(), + "custom_llm_provider": request.custom_llm_provider(), + "url": request.url(), + "body": request.body(), })), ) .await @@ -158,7 +142,7 @@ impl AudioTranscriptionLifecycleHooks { let body = data.remove("body").ok_or_else(|| { Error::InvalidRequest("audio transcription guardrail removed body".to_string()) })?; - Ok(ProviderAudioTranscriptionRequest { body, ..request }) + Ok(request.with_body(body)) } fn logging_payload( diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs index 3983846d7b6..03d621b8414 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs @@ -1,16 +1,14 @@ use litellm_core::Error; +use litellm_core::audio_transcription::execute_audio_transcription_provider_call; use litellm_core::call_lifecycle::CallLifecycle; use serde_json::Value; -mod common_utils; -mod handler; mod hooks; mod prepare; mod types; pub use types::AudioTranscriptionRequest; -use handler::execute_audio_transcription_provider_call; use prepare::{PreparedAudioTranscriptionCall, prepare_audio_transcription_call}; pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs index 9697aa98b0a..b470638264e 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs @@ -1,7 +1,6 @@ use std::sync::Arc; use std::time::Duration; -use litellm_core::audio_transcription::transformation::AudioTranscriptionProviderConfig; use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleRequest}; use serde_json::{Map, Value}; @@ -46,13 +45,3 @@ impl CallLifecycleRequest for PreparedAudioTranscriptionRequest { ) } } - -#[derive(Clone)] -pub(crate) struct ProviderAudioTranscriptionRequest { - pub(crate) model: String, - pub(crate) config: &'static dyn AudioTranscriptionProviderConfig, - pub(crate) url: String, - pub(crate) body: Value, - pub(crate) upstream_headers: Vec<(String, String)>, - pub(crate) timeout: Option, -} diff --git a/litellm-rust/crates/core/src/audio_transcription/client.rs b/litellm-rust/crates/core/src/audio_transcription/client.rs new file mode 100644 index 00000000000..0e612628dc6 --- /dev/null +++ b/litellm-rust/crates/core/src/audio_transcription/client.rs @@ -0,0 +1,14 @@ +use std::sync::OnceLock; +use std::time::Duration; + +use crate::constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS; + +pub(super) fn http_client() -> &'static reqwest::Client { + static CLIENT: OnceLock = OnceLock::new(); + CLIENT.get_or_init(|| { + reqwest::Client::builder() + .timeout(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()) + }) +} diff --git a/litellm-rust/crates/core/src/audio_transcription/handler.rs b/litellm-rust/crates/core/src/audio_transcription/handler.rs new file mode 100644 index 00000000000..30ba0da5e68 --- /dev/null +++ b/litellm-rust/crates/core/src/audio_transcription/handler.rs @@ -0,0 +1,91 @@ +use serde_json::Value; + +use crate::error::Error; +use crate::http_utils::truncate_error_body; + +use super::client::http_client; +use super::types::ProviderAudioTranscriptionRequest; + +pub async fn execute_audio_transcription_provider_call( + request: ProviderAudioTranscriptionRequest, +) -> Result { + let body = serde_json::to_vec(&request.body) + .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?; + let headers = signed_headers(&request, &body).await?; + let mut request_builder = http_client().post(&request.url).body(body); + for (key, value) in headers { + request_builder = request_builder.header(key, value); + } + if let Some(duration) = request.timeout { + request_builder = request_builder.timeout(duration); + } + let response = request_builder + .send() + .await + .map_err(|error| Error::Network(error.to_string()))?; + let status = response.status(); + let text = response + .text() + .await + .map_err(|error| Error::Network(error.to_string()))?; + if !status.is_success() { + return Err(Error::Http { + status: status.as_u16(), + body: truncate_error_body(&text), + }); + } + let response_json = serde_json::from_str(&text) + .map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?; + Ok(request + .config + .transform_transcription_response(&request.model, response_json)? + .into_json()) +} + +#[cfg(feature = "bedrock-auth")] +async fn signed_headers( + request: &ProviderAudioTranscriptionRequest, + body: &[u8], +) -> Result, Error> { + use std::collections::BTreeMap; + use std::time::SystemTime; + + use crate::audio_transcription::transformation::AudioTranscriptionAuth; + use crate::providers::bedrock::audio_transcription::aws_auth_config; + use crate::providers::bedrock::aws_base::{resolve_credentials, sign_bedrock_post}; + + let AudioTranscriptionAuth::AwsSigV4 { region, .. } = &request.auth else { + return Ok(request.upstream_headers.clone()); + }; + let env_lookup = |key: &str| std::env::var(key).ok(); + let credentials = resolve_credentials( + aws_auth_config(&request.optional_params, &env_lookup), + &env_lookup, + ) + .await?; + let unsigned: BTreeMap = request.upstream_headers.iter().cloned().collect(); + let signature = sign_bedrock_post( + &request.url, + body, + &unsigned, + region, + &credentials, + SystemTime::now(), + )?; + Ok(unsigned.into_iter().chain(signature).collect()) +} + +#[cfg(not(feature = "bedrock-auth"))] +async fn signed_headers( + request: &ProviderAudioTranscriptionRequest, + _body: &[u8], +) -> Result, Error> { + use crate::audio_transcription::transformation::AudioTranscriptionAuth; + + match request.auth { + AudioTranscriptionAuth::AwsSigV4 { .. } => Err(Error::Unsupported( + "AWS SigV4 requires the bedrock-auth feature", + )), + AudioTranscriptionAuth::Bearer => Ok(request.upstream_headers.clone()), + } +} diff --git a/litellm-rust/crates/core/src/audio_transcription/mod.rs b/litellm-rust/crates/core/src/audio_transcription/mod.rs index ec2fbb969a6..b71748082bf 100644 --- a/litellm-rust/crates/core/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/core/src/audio_transcription/mod.rs @@ -1,2 +1,20 @@ +use crate::Error; +mod client; +mod handler; +mod prepare; pub mod transformation; pub mod types; + +use serde_json::Value; + +pub use handler::execute_audio_transcription_provider_call; +pub use prepare::prepare_audio_transcription_provider_call; +pub use types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; + +pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { + execute_audio_transcription_provider_call(prepare_audio_transcription_provider_call(request)?) + .await +} + +#[cfg(test)] +mod tests; diff --git a/litellm-rust/crates/core/src/audio_transcription/prepare.rs b/litellm-rust/crates/core/src/audio_transcription/prepare.rs new file mode 100644 index 00000000000..6288e96b380 --- /dev/null +++ b/litellm-rust/crates/core/src/audio_transcription/prepare.rs @@ -0,0 +1,72 @@ +use crate::error::Error; +use crate::http_utils::{has_header, string_headers}; +#[cfg(feature = "bedrock-auth")] +use crate::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG; +use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; + +use super::transformation::{AudioTranscriptionAuth, AudioTranscriptionProviderConfig}; +use super::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; + +fn provider_config(provider: &str) -> Option<&'static dyn AudioTranscriptionProviderConfig> { + #[cfg(feature = "bedrock-auth")] + if provider == "bedrock" { + return Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG); + } + let _ = provider; + None +} + +pub fn prepare_audio_transcription_provider_call( + request: AudioTranscriptionRequest<'_>, +) -> Result { + let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider) + .or_else(|| { + request + .custom_llm_provider + .map(|provider| CustomLlmProvider { + model: request.model, + custom_llm_provider: provider, + }) + }) + .ok_or_else(|| { + Error::InvalidProvider( + "unable to resolve custom_llm_provider for audio transcription request".to_string(), + ) + })?; + let model = provider_info.model.to_string(); + let config = provider_config(provider_info.custom_llm_provider) + .ok_or_else(|| Error::InvalidProvider(provider_info.custom_llm_provider.to_string()))?; + let env_lookup = |key: &str| std::env::var(key).ok(); + let mut headers = string_headers("audio transcription", request.extra_headers)?; + let auth = config.auth_strategy(&model, &request.optional_params, &env_lookup)?; + if matches!(auth, AudioTranscriptionAuth::Bearer) + && !has_header(&headers, "authorization") + && let Some(api_key) = request.api_key + { + headers.push(("Authorization".to_string(), format!("Bearer {api_key}"))); + } + if !has_header(&headers, "content-type") { + headers.push(("Content-Type".to_string(), "application/json".to_string())); + } + let url = config.complete_url( + request.api_base, + &model, + &request.optional_params, + &env_lookup, + )?; + let filtered_params = config.map_transcription_params(&request.optional_params); + let transformed = + config.transform_transcription_request(&model, request.audio, filtered_params)?; + Ok(ProviderAudioTranscriptionRequest { + model, + custom_llm_provider: provider_info.custom_llm_provider.to_string(), + config, + url, + body: transformed.body, + upstream_headers: headers, + auth, + #[cfg(feature = "bedrock-auth")] + optional_params: request.optional_params, + timeout: request.timeout, + }) +} diff --git a/litellm-rust/crates/core/src/audio_transcription/tests.rs b/litellm-rust/crates/core/src/audio_transcription/tests.rs new file mode 100644 index 00000000000..263d63337b0 --- /dev/null +++ b/litellm-rust/crates/core/src/audio_transcription/tests.rs @@ -0,0 +1,50 @@ +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::thread; + +use serde_json::{Map, json}; + +use super::audio_transcription; +use super::types::AudioTranscriptionRequest; + +#[tokio::test] +async fn bedrock_request_is_signed_and_contains_audio() { + let listener = TcpListener::bind("127.0.0.1:0").expect("listener"); + let address = listener.local_addr().expect("address"); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("connection"); + let mut request = Vec::new(); + let mut buffer = [0_u8; 16_384]; + let count = stream.read(&mut buffer).expect("request"); + request.extend_from_slice(&buffer[..count]); + let request = String::from_utf8_lossy(&request); + assert!(request.contains("POST /model/mistral.voxtral-mini-3b-2507/converse")); + assert!(request.contains("authorization: AWS4-HMAC-SHA256")); + assert!(request.contains("x-amz-date:")); + assert!(request.contains("\"bytes\":\"AQI=\"")); + assert!(request.contains("Transcribe the audio. Respond with only the transcript.")); + let response = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 53\r\nConnection: close\r\n\r\n{\"output\":{\"message\":{\"content\":[{\"text\":\"hello\"}]}}}"; + stream.write_all(response).expect("response"); + }); + + let optional_params = Map::from_iter([ + ("aws_access_key_id".to_string(), json!("access-key")), + ("aws_secret_access_key".to_string(), json!("secret-key")), + ("aws_region_name".to_string(), json!("us-east-1")), + ]); + let api_base = format!("http://{address}"); + let response = audio_transcription(AudioTranscriptionRequest { + model: "mistral.voxtral-mini-3b-2507", + audio: json!({"data": "AQI=", "format": "wav", "filename": "audio.wav"}), + api_key: None, + api_base: Some(&api_base), + custom_llm_provider: Some("bedrock"), + extra_headers: None, + optional_params, + timeout: None, + }) + .await + .expect("transcription"); + assert_eq!(response, json!({"text": "hello"})); + server.join().expect("server"); +} diff --git a/litellm-rust/crates/core/src/audio_transcription/types.rs b/litellm-rust/crates/core/src/audio_transcription/types.rs index 3a9e1ecd88c..559d7837027 100644 --- a/litellm-rust/crates/core/src/audio_transcription/types.rs +++ b/litellm-rust/crates/core/src/audio_transcription/types.rs @@ -1,5 +1,56 @@ +use std::time::Duration; + use serde::{Deserialize, Serialize}; -use serde_json::Value; +use serde_json::{Map, Value}; + +use super::transformation::{AudioTranscriptionAuth, AudioTranscriptionProviderConfig}; + +pub struct AudioTranscriptionRequest<'a> { + pub model: &'a str, + pub audio: Value, + pub api_key: Option<&'a str>, + pub api_base: Option<&'a str>, + pub custom_llm_provider: Option<&'a str>, + pub extra_headers: Option>, + pub optional_params: Map, + pub timeout: Option, +} + +#[derive(Clone)] +pub struct ProviderAudioTranscriptionRequest { + pub(super) model: String, + pub(super) custom_llm_provider: String, + pub(super) config: &'static dyn AudioTranscriptionProviderConfig, + pub(super) url: String, + pub(super) body: Value, + pub(super) upstream_headers: Vec<(String, String)>, + pub(super) auth: AudioTranscriptionAuth, + #[cfg(feature = "bedrock-auth")] + pub(super) optional_params: Map, + pub(super) timeout: Option, +} + +impl ProviderAudioTranscriptionRequest { + pub fn model(&self) -> &str { + &self.model + } + + pub fn custom_llm_provider(&self) -> &str { + &self.custom_llm_provider + } + + pub fn url(&self) -> &str { + &self.url + } + + pub fn body(&self) -> &Value { + &self.body + } + + pub fn with_body(self, body: Value) -> Self { + Self { body, ..self } + } +} #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct AudioTranscriptionRequestData { diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index e1ac0a4fc8f..a73961060eb 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -30,6 +30,8 @@ pub(crate) const CHAT_COMPLETIONS_TIMEOUT_SECS: u64 = 600; /// Connect timeout for chat completions provider calls, in seconds. pub(crate) const CHAT_COMPLETIONS_CONNECT_TIMEOUT_SECS: u64 = 10; +pub(crate) const AUDIO_TRANSCRIPTION_TIMEOUT_SECS: u64 = 600; + /// `object` field every non-streaming chat completion response carries. pub const CHAT_COMPLETION_OBJECT: &str = "chat.completion"; diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 68aa9436b15..2e2624acbe1 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -1,11 +1,11 @@ use std::collections::HashMap; use std::time::Duration; -use litellm_ai_gateway::io::audio_transcription::{ - AudioTranscriptionRequest, audio_transcription as run_audio_transcription, -}; use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr}; use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; +use litellm_core::audio_transcription::{ + AudioTranscriptionRequest, audio_transcription as run_audio_transcription, +}; use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse}; use litellm_core::chat_completions::{ chat_completions as run_chat_completions, chat_completions_decline_reason, @@ -325,10 +325,6 @@ fn transcription( extra_headers, optional_params, timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, }, )) }); @@ -369,10 +365,6 @@ fn atranscription( extra_headers, optional_params, timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, }) .await .map_err(core_error_to_pyerr)?; From ece5dd226e0af7bab8ef87c4329c9157d21c34ba Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 15:36:42 -0700 Subject: [PATCH 54/67] refactor(python-bridge): split non-streaming bridge modules --- .../crates/python-bridge/src/diagnostics.rs | 23 + .../crates/python-bridge/src/errors.rs | 61 ++ litellm-rust/crates/python-bridge/src/lib.rs | 621 ++---------------- .../crates/python-bridge/src/marshal.rs | 53 ++ .../src/routes/audio_transcription.rs | 92 +++ .../src/routes/chat_completions.rs | 163 +++++ .../python-bridge/src/routes/messages.rs | 109 +++ .../crates/python-bridge/src/routes/mod.rs | 13 + .../crates/python-bridge/src/routes/ocr.rs | 128 ++++ 9 files changed, 690 insertions(+), 573 deletions(-) create mode 100644 litellm-rust/crates/python-bridge/src/diagnostics.rs create mode 100644 litellm-rust/crates/python-bridge/src/errors.rs create mode 100644 litellm-rust/crates/python-bridge/src/marshal.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/chat_completions.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/messages.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/mod.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/ocr.rs diff --git a/litellm-rust/crates/python-bridge/src/diagnostics.rs b/litellm-rust/crates/python-bridge/src/diagnostics.rs new file mode 100644 index 00000000000..cc153a89b8f --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/diagnostics.rs @@ -0,0 +1,23 @@ +use litellm_python_interop::release_count; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +#[pyfunction] +fn gil_stats(py: Python<'_>) -> PyResult> { + let stats = PyDict::new(py); + stats.set_item("releases", release_count())?; + Ok(stats.into_any().unbind()) +} + +#[cfg(feature = "panic-test")] +#[pyfunction] +fn _panic_for_test() { + panic!("intentional PyO3 panic smoke test"); +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(gil_stats, module)?)?; + #[cfg(feature = "panic-test")] + module.add_function(wrap_pyfunction!(_panic_for_test, module)?)?; + Ok(()) +} diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs new file mode 100644 index 00000000000..914e2e1e033 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -0,0 +1,61 @@ +use litellm_core::error::Error; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::*; + +pyo3::create_exception!( + _native, + RustBridgeDeclined, + pyo3::exceptions::PyException, + "The route declined before calling the provider, so the host may retry on its own path." +); + +pyo3::create_exception!( + _native, + RustUpstreamError, + pyo3::exceptions::PyException, + "The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response." +); + +pub(crate) fn core_error_to_pyerr(err: Error) -> PyErr { + match err { + Error::Auth(message) => PyValueError::new_err(message), + Error::InvalidProvider(_) + | Error::InvalidRequest(_) + | Error::InvalidType { .. } + | Error::MissingField(_) => PyValueError::new_err(err.to_string()), + other => PyRuntimeError::new_err(other.to_string()), + } +} + +/// Map a core error for a route whose host keeps a Python implementation. +/// +/// The distinction the host needs is whether the provider was already called. +/// Everything raised before the request goes out is safe for the host to retry +/// on its own path; anything after it is not, because the provider has already +/// done the work and billed for it. +pub(crate) fn chat_completions_error_to_pyerr(err: Error) -> PyErr { + match err { + Error::Unsupported(_) + | Error::Auth(_) + | Error::InvalidProvider(_) + | Error::InvalidRequest(_) + | Error::InvalidType { .. } + | Error::MissingField(_) + | Error::Routing(_) + // Nothing reached the provider, so serving it on Python cannot double + // bill and is the only way the caller gets an answer at all. + | Error::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), + Error::Http { status, body } => { + RustUpstreamError::new_err((status, format!("{status}: {body}"))) + } + Error::Network(message) | Error::InvalidResponse(message) => { + RustUpstreamError::new_err((0u16, message)) + } + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + module.add("RustBridgeDeclined", py.get_type::())?; + module.add("RustUpstreamError", py.get_type::()) +} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 2e2624acbe1..f628c987220 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -1,142 +1,14 @@ -use std::collections::HashMap; -use std::time::Duration; +mod diagnostics; +mod errors; +mod marshal; +mod routes; -use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr}; use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; -use litellm_core::audio_transcription::{ - AudioTranscriptionRequest, audio_transcription as run_audio_transcription, -}; -use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse}; -use litellm_core::chat_completions::{ - chat_completions as run_chat_completions, chat_completions_decline_reason, -}; -use litellm_core::error::Error; -use litellm_core::messages::messages as run_messages; -use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; -use litellm_python_interop::{from_py, release_count, release_gil, to_py}; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::{PyAny, PyDict}; -use serde_json::{Map, Value}; +use pyo3::types::PyAny; -pyo3::create_exception!( - _native, - RustBridgeDeclined, - pyo3::exceptions::PyException, - "The route declined before calling the provider, so the host may retry on its own path." -); - -pyo3::create_exception!( - _native, - RustUpstreamError, - pyo3::exceptions::PyException, - "The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response." -); - -type MarshaledOcrInputs = ( - Value, - Option>, - Map, - Option, -); - -fn messages_response_to_py( - py: Python<'_>, - response: AnthropicMessagesResponse, -) -> PyResult> { - to_py(py, &response) -} - -fn chat_completions_response_to_py( - py: Python<'_>, - response: ChatCompletionsResponse, -) -> PyResult> { - to_py(py, &response) -} - -fn core_error_to_pyerr(err: Error) -> PyErr { - match err { - Error::Auth(message) => PyValueError::new_err(message), - Error::InvalidProvider(_) - | Error::InvalidRequest(_) - | Error::InvalidType { .. } - | Error::MissingField(_) => PyValueError::new_err(err.to_string()), - other => PyRuntimeError::new_err(other.to_string()), - } -} - -/// Map a core error for a route whose host keeps a Python implementation. -/// -/// The distinction the host needs is whether the provider was already called. -/// Everything raised before the request goes out is safe for the host to retry -/// on its own path; anything after it is not, because the provider has already -/// done the work and billed for it. -fn chat_completions_error_to_pyerr(err: Error) -> PyErr { - match err { - Error::Unsupported(_) - | Error::Auth(_) - | Error::InvalidProvider(_) - | Error::InvalidRequest(_) - | Error::InvalidType { .. } - | Error::MissingField(_) - | Error::Routing(_) - // Nothing reached the provider, so serving it on Python cannot double - // bill and is the only way the caller gets an answer at all. - | Error::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), - Error::Http { status, body } => { - RustUpstreamError::new_err((status, format!("{status}: {body}"))) - } - Error::Network(message) | Error::InvalidResponse(message) => { - RustUpstreamError::new_err((0u16, message)) - } - } -} - -fn optional_object_to_map( - py: Python<'_>, - name: &'static str, - value: Option>, -) -> PyResult> { - match value { - Some(value) => match from_py(value.bind(py))? { - Value::Object(map) => Ok(map), - _ => Err(PyValueError::new_err(format!("{name} must be a dict"))), - }, - None => Ok(Map::new()), - } -} - -fn optional_timeout(timeout_seconds: Option) -> Option { - timeout_seconds.and_then(|secs| { - if secs.is_finite() && secs > 0.0 { - Some(Duration::from_secs_f64(secs)) - } else { - None - } - }) -} - -fn marshal_headers( - py: Python<'_>, - headers: Option>, -) -> PyResult> { - let value = match headers { - Some(headers) => from_py(headers.bind(py))?, - None => Value::Object(Map::new()), - }; - let Value::Object(headers) = value else { - return Err(PyValueError::new_err("headers must be a dict")); - }; - headers - .into_iter() - .map(|(name, value)| { - value - .as_str() - .map(|value| (name, value.to_string())) - .ok_or_else(|| PyValueError::new_err("header values must be strings")) - }) - .collect() -} +use crate::errors::core_error_to_pyerr; +use crate::marshal::{marshal_headers, optional_timeout}; #[pyclass] struct ResponsesWebSocketConnection { @@ -186,445 +58,48 @@ impl ResponsesWebSocketConnection { } } -fn marshal_inputs( - py: Python<'_>, - document: Py, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult { - let document = from_py(document.bind(py))?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let timeout = optional_timeout(timeout_seconds); - - Ok((document, extra_headers, optional_params, timeout)) -} - -#[pyfunction] -#[pyo3(signature = (model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn ocr( - py: Python<'_>, - model: String, - document: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (document, extra_headers, optional_params, timeout) = marshal_inputs( - py, - document, - extra_headers, - optional_params, - timeout_seconds, - )?; - - let result = release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_ocr(OcrRequest { - model: &model, - document, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - })) - }); - - match result { - Ok(value) => to_py(py, &value), - Err(err) => Err(core_error_to_pyerr(err)), - } -} - -#[pyfunction] -#[pyo3(signature = (model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn aocr( - py: Python<'_>, - model: String, - document: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (document, extra_headers, optional_params, timeout) = marshal_inputs( - py, - document, - extra_headers, - optional_params, - timeout_seconds, - )?; - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let value = run_ocr(OcrRequest { - model: &model, - document, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - }) - .await - .map_err(core_error_to_pyerr)?; - - Python::attach(|py| to_py(py, &value)) - }) -} - -#[pyfunction] -#[pyo3(signature = (model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn transcription( - py: Python<'_>, - model: String, - audio: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let audio = from_py(audio.bind(py))?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let timeout = optional_timeout(timeout_seconds); - let result = release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_audio_transcription( - AudioTranscriptionRequest { - model: &model, - audio, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - }, - )) - }); - match result { - Ok(value) => to_py(py, &value), - Err(err) => Err(core_error_to_pyerr(err)), - } -} - -#[pyfunction] -#[pyo3(signature = (model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn atranscription( - py: Python<'_>, - model: String, - audio: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let audio = from_py(audio.bind(py))?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let timeout = optional_timeout(timeout_seconds); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let value = run_audio_transcription(AudioTranscriptionRequest { - model: &model, - audio, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - }) - .await - .map_err(core_error_to_pyerr)?; - Python::attach(|py| to_py(py, &value)) - }) -} - -type MarshaledMessagesInputs = (Value, Option>, Option); - -fn marshal_messages_inputs( - py: Python<'_>, - body: Py, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult { - let body: Value = from_py(body.bind(py))?; - if !body.is_object() { - return Err(PyValueError::new_err("body must be a dict")); - } - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - Ok((body, extra_headers, optional_timeout(timeout_seconds))) -} - -#[pyfunction] -#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn messages( - py: Python<'_>, - model: String, - body: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (body, extra_headers, timeout) = - marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; - - let result = release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_messages(MessagesRequest { - model: &model, - body, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - })) - }); - - match result { - Ok(response) => messages_response_to_py(py, response), - Err(err) => Err(core_error_to_pyerr(err)), - } -} - -#[pyfunction] -#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn amessages( - py: Python<'_>, - model: String, - body: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (body, extra_headers, timeout) = - marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let response = run_messages(MessagesRequest { - model: &model, - body, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }) - .await - .map_err(core_error_to_pyerr)?; - - Python::attach(|py| messages_response_to_py(py, response)) - }) -} - -type MarshaledChatCompletionsInputs = ( - Value, - Map, - Option>, - Option, -); - -fn marshal_chat_completions_inputs( - py: Python<'_>, - messages: Py, - optional_params: Option>, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult { - let messages: Value = from_py(messages.bind(py))?; - if !messages.is_array() { - return Err(PyValueError::new_err("messages must be a list")); - } - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - Ok(( - messages, - optional_params, - extra_headers, - optional_timeout(timeout_seconds), - )) -} - -/// The decline reason for this request, or `None` when the Rust path accepts -/// it. Resolves no credentials and performs no I/O, so a host can ask before -/// committing to either path. -#[pyfunction] -#[pyo3(signature = (model, messages, optional_params=None, custom_llm_provider=None))] -fn chat_completions_decline( - py: Python<'_>, - model: String, - messages: Py, - optional_params: Option>, - custom_llm_provider: Option, -) -> PyResult> { - let messages = from_py(messages.bind(py))?; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - Ok(chat_completions_decline_reason( - &model, - custom_llm_provider.as_deref(), - messages, - &optional_params, - ) - .map(str::to_string)) -} - -#[pyfunction] -#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn chat_completions( - py: Python<'_>, - model: String, - messages: Py, - optional_params: Option>, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (messages, optional_params, extra_headers, timeout) = marshal_chat_completions_inputs( - py, - messages, - optional_params, - extra_headers, - timeout_seconds, - )?; - - let result = release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_chat_completions( - ChatCompletionsRequest { - model: &model, - messages, - optional_params, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }, - )) - }); - - match result { - Ok(response) => chat_completions_response_to_py(py, response), - Err(err) => Err(chat_completions_error_to_pyerr(err)), - } -} - -#[pyfunction] -#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn achat_completions( - py: Python<'_>, - model: String, - messages: Py, - optional_params: Option>, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (messages, optional_params, extra_headers, timeout) = marshal_chat_completions_inputs( - py, - messages, - optional_params, - extra_headers, - timeout_seconds, - )?; - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let response = run_chat_completions(ChatCompletionsRequest { - model: &model, - messages, - optional_params, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }) - .await - .map_err(chat_completions_error_to_pyerr)?; - - Python::attach(|py| chat_completions_response_to_py(py, response)) - }) -} - -#[pyfunction] -fn gil_stats(py: Python<'_>) -> PyResult> { - let stats = PyDict::new(py); - stats.set_item("releases", release_count())?; - Ok(stats.into_any().unbind()) -} - -#[cfg(feature = "panic-test")] -#[pyfunction] -fn _panic_for_test() { - panic!("intentional PyO3 panic smoke test"); -} - #[pymodule] fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py = module.py(); - module.add_function(wrap_pyfunction!(ocr, module)?)?; - module.add_function(wrap_pyfunction!(aocr, module)?)?; - module.add_function(wrap_pyfunction!(transcription, module)?)?; - module.add_function(wrap_pyfunction!(atranscription, module)?)?; - module.add_function(wrap_pyfunction!(messages, module)?)?; - module.add_function(wrap_pyfunction!(amessages, module)?)?; - module.add("RustBridgeDeclined", py.get_type::())?; - module.add("RustUpstreamError", py.get_type::())?; - module.add_function(wrap_pyfunction!(chat_completions_decline, module)?)?; - module.add_function(wrap_pyfunction!(chat_completions, module)?)?; - module.add_function(wrap_pyfunction!(achat_completions, module)?)?; + errors::register(module)?; + routes::register(module)?; module.add_class::()?; - module.add_function(wrap_pyfunction!(gil_stats, module)?)?; - #[cfg(feature = "panic-test")] - module.add_function(wrap_pyfunction!(_panic_for_test, module)?)?; - Ok(()) + diagnostics::register(module) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn module_registration_preserves_the_public_surface() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "_native").expect("module should be created"); + _native(&module).expect("module should register"); + + let expected = [ + "RustBridgeDeclined", + "RustUpstreamError", + "ocr", + "aocr", + "transcription", + "atranscription", + "messages", + "amessages", + "chat_completions_decline", + "chat_completions", + "achat_completions", + "ResponsesWebSocketConnection", + "gil_stats", + ]; + + for name in expected { + assert!( + module + .hasattr(name) + .expect("attribute lookup should succeed") + ); + } + }); + } } diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs new file mode 100644 index 00000000000..6c070d2a9ee --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -0,0 +1,53 @@ +use std::collections::HashMap; +use std::time::Duration; + +use litellm_python_interop::from_py; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use serde_json::{Map, Value}; + +pub(crate) fn optional_object_to_map( + py: Python<'_>, + name: &'static str, + value: Option>, +) -> PyResult> { + match value { + Some(value) => match from_py(value.bind(py))? { + Value::Object(map) => Ok(map), + _ => Err(PyValueError::new_err(format!("{name} must be a dict"))), + }, + None => Ok(Map::new()), + } +} + +pub(crate) fn optional_timeout(timeout_seconds: Option) -> Option { + timeout_seconds.and_then(|secs| { + if secs.is_finite() && secs > 0.0 { + Some(Duration::from_secs_f64(secs)) + } else { + None + } + }) +} + +pub(crate) fn marshal_headers( + py: Python<'_>, + headers: Option>, +) -> PyResult> { + let value = match headers { + Some(headers) => from_py(headers.bind(py))?, + None => Value::Object(Map::new()), + }; + let Value::Object(headers) = value else { + return Err(PyValueError::new_err("headers must be a dict")); + }; + headers + .into_iter() + .map(|(name, value)| { + value + .as_str() + .map(|value| (name, value.to_string())) + .ok_or_else(|| PyValueError::new_err("header values must be strings")) + }) + .collect() +} diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs new file mode 100644 index 00000000000..4017cf0887e --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs @@ -0,0 +1,92 @@ +use litellm_core::audio_transcription::{ + AudioTranscriptionRequest, audio_transcription as run_audio_transcription, +}; +use litellm_python_interop::{from_py, release_gil, to_py}; +use pyo3::prelude::*; + +use crate::errors::core_error_to_pyerr; +use crate::marshal::{optional_object_to_map, optional_timeout}; + +#[pyfunction] +#[pyo3(signature = (model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] +#[allow(clippy::too_many_arguments)] +fn transcription( + py: Python<'_>, + model: String, + audio: Py, + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + extra_headers: Option>, + optional_params: Option>, + timeout_seconds: Option, +) -> PyResult> { + let audio = from_py(audio.bind(py))?; + let extra_headers = match extra_headers { + Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), + None => None, + }; + let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; + let timeout = optional_timeout(timeout_seconds); + let result = release_gil(py, || { + pyo3_async_runtimes::tokio::get_runtime().block_on(run_audio_transcription( + AudioTranscriptionRequest { + model: &model, + audio, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + }, + )) + }); + match result { + Ok(value) => to_py(py, &value), + Err(err) => Err(core_error_to_pyerr(err)), + } +} + +#[pyfunction] +#[pyo3(signature = (model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] +#[allow(clippy::too_many_arguments)] +fn atranscription( + py: Python<'_>, + model: String, + audio: Py, + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + extra_headers: Option>, + optional_params: Option>, + timeout_seconds: Option, +) -> PyResult> { + let audio = from_py(audio.bind(py))?; + let extra_headers = match extra_headers { + Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), + None => None, + }; + let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; + let timeout = optional_timeout(timeout_seconds); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let value = run_audio_transcription(AudioTranscriptionRequest { + model: &model, + audio, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + }) + .await + .map_err(core_error_to_pyerr)?; + Python::attach(|py| to_py(py, &value)) + }) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(transcription, module)?)?; + module.add_function(wrap_pyfunction!(atranscription, module)?) +} diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs new file mode 100644 index 00000000000..b353765e7cc --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs @@ -0,0 +1,163 @@ +use std::time::Duration; + +use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse}; +use litellm_core::chat_completions::{ + chat_completions as run_chat_completions, chat_completions_decline_reason, +}; +use litellm_python_interop::{from_py, release_gil, to_py}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use serde_json::{Map, Value}; + +use crate::errors::chat_completions_error_to_pyerr; +use crate::marshal::{optional_object_to_map, optional_timeout}; + +fn chat_completions_response_to_py( + py: Python<'_>, + response: ChatCompletionsResponse, +) -> PyResult> { + to_py(py, &response) +} + +type MarshaledChatCompletionsInputs = ( + Value, + Map, + Option>, + Option, +); + +fn marshal_chat_completions_inputs( + py: Python<'_>, + messages: Py, + optional_params: Option>, + extra_headers: Option>, + timeout_seconds: Option, +) -> PyResult { + let messages: Value = from_py(messages.bind(py))?; + if !messages.is_array() { + return Err(PyValueError::new_err("messages must be a list")); + } + let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; + let extra_headers = match extra_headers { + Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), + None => None, + }; + Ok(( + messages, + optional_params, + extra_headers, + optional_timeout(timeout_seconds), + )) +} + +/// The decline reason for this request, or `None` when the Rust path accepts +/// it. Resolves no credentials and performs no I/O, so a host can ask before +/// committing to either path. +#[pyfunction] +#[pyo3(signature = (model, messages, optional_params=None, custom_llm_provider=None))] +fn chat_completions_decline( + py: Python<'_>, + model: String, + messages: Py, + optional_params: Option>, + custom_llm_provider: Option, +) -> PyResult> { + let messages = from_py(messages.bind(py))?; + let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; + Ok(chat_completions_decline_reason( + &model, + custom_llm_provider.as_deref(), + messages, + &optional_params, + ) + .map(str::to_string)) +} + +#[pyfunction] +#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] +#[allow(clippy::too_many_arguments)] +fn chat_completions( + py: Python<'_>, + model: String, + messages: Py, + optional_params: Option>, + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + extra_headers: Option>, + timeout_seconds: Option, +) -> PyResult> { + let (messages, optional_params, extra_headers, timeout) = marshal_chat_completions_inputs( + py, + messages, + optional_params, + extra_headers, + timeout_seconds, + )?; + + let result = release_gil(py, || { + pyo3_async_runtimes::tokio::get_runtime().block_on(run_chat_completions( + ChatCompletionsRequest { + model: &model, + messages, + optional_params, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + }, + )) + }); + + match result { + Ok(response) => chat_completions_response_to_py(py, response), + Err(err) => Err(chat_completions_error_to_pyerr(err)), + } +} + +#[pyfunction] +#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] +#[allow(clippy::too_many_arguments)] +fn achat_completions( + py: Python<'_>, + model: String, + messages: Py, + optional_params: Option>, + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + extra_headers: Option>, + timeout_seconds: Option, +) -> PyResult> { + let (messages, optional_params, extra_headers, timeout) = marshal_chat_completions_inputs( + py, + messages, + optional_params, + extra_headers, + timeout_seconds, + )?; + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let response = run_chat_completions(ChatCompletionsRequest { + model: &model, + messages, + optional_params, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + }) + .await + .map_err(chat_completions_error_to_pyerr)?; + + Python::attach(|py| chat_completions_response_to_py(py, response)) + }) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(chat_completions_decline, module)?)?; + module.add_function(wrap_pyfunction!(chat_completions, module)?)?; + module.add_function(wrap_pyfunction!(achat_completions, module)?) +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages.rs b/litellm-rust/crates/python-bridge/src/routes/messages.rs new file mode 100644 index 00000000000..ac361bb0e59 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages.rs @@ -0,0 +1,109 @@ +use std::time::Duration; + +use litellm_core::messages::messages as run_messages; +use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; +use litellm_python_interop::{from_py, release_gil, to_py}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use serde_json::{Map, Value}; + +use crate::errors::core_error_to_pyerr; +use crate::marshal::{optional_object_to_map, optional_timeout}; + +fn messages_response_to_py( + py: Python<'_>, + response: AnthropicMessagesResponse, +) -> PyResult> { + to_py(py, &response) +} + +type MarshaledMessagesInputs = (Value, Option>, Option); + +fn marshal_messages_inputs( + py: Python<'_>, + body: Py, + extra_headers: Option>, + timeout_seconds: Option, +) -> PyResult { + let body: Value = from_py(body.bind(py))?; + if !body.is_object() { + return Err(PyValueError::new_err("body must be a dict")); + } + let extra_headers = match extra_headers { + Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), + None => None, + }; + Ok((body, extra_headers, optional_timeout(timeout_seconds))) +} + +#[pyfunction] +#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] +#[allow(clippy::too_many_arguments)] +fn messages( + py: Python<'_>, + model: String, + body: Py, + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + extra_headers: Option>, + timeout_seconds: Option, +) -> PyResult> { + let (body, extra_headers, timeout) = + marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; + + let result = release_gil(py, || { + pyo3_async_runtimes::tokio::get_runtime().block_on(run_messages(MessagesRequest { + model: &model, + body, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + })) + }); + + match result { + Ok(response) => messages_response_to_py(py, response), + Err(err) => Err(core_error_to_pyerr(err)), + } +} + +#[pyfunction] +#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] +#[allow(clippy::too_many_arguments)] +fn amessages( + py: Python<'_>, + model: String, + body: Py, + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + extra_headers: Option>, + timeout_seconds: Option, +) -> PyResult> { + let (body, extra_headers, timeout) = + marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let response = run_messages(MessagesRequest { + model: &model, + body, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + }) + .await + .map_err(core_error_to_pyerr)?; + + Python::attach(|py| messages_response_to_py(py, response)) + }) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(messages, module)?)?; + module.add_function(wrap_pyfunction!(amessages, module)?) +} diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs new file mode 100644 index 00000000000..a2eb8355767 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -0,0 +1,13 @@ +use pyo3::prelude::*; + +mod audio_transcription; +mod chat_completions; +mod messages; +mod ocr; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + ocr::register(module)?; + audio_transcription::register(module)?; + messages::register(module)?; + chat_completions::register(module) +} diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr.rs b/litellm-rust/crates/python-bridge/src/routes/ocr.rs new file mode 100644 index 00000000000..047bf245d7c --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/ocr.rs @@ -0,0 +1,128 @@ +use std::time::Duration; + +use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr}; +use litellm_python_interop::{from_py, release_gil, to_py}; +use pyo3::prelude::*; +use serde_json::{Map, Value}; + +use crate::errors::core_error_to_pyerr; +use crate::marshal::{optional_object_to_map, optional_timeout}; + +type MarshaledOcrInputs = ( + Value, + Option>, + Map, + Option, +); + +fn marshal_inputs( + py: Python<'_>, + document: Py, + extra_headers: Option>, + optional_params: Option>, + timeout_seconds: Option, +) -> PyResult { + let document = from_py(document.bind(py))?; + let extra_headers = match extra_headers { + Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), + None => None, + }; + let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; + let timeout = optional_timeout(timeout_seconds); + + Ok((document, extra_headers, optional_params, timeout)) +} + +#[pyfunction] +#[pyo3(signature = (model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] +#[allow(clippy::too_many_arguments)] +fn ocr( + py: Python<'_>, + model: String, + document: Py, + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + extra_headers: Option>, + optional_params: Option>, + timeout_seconds: Option, +) -> PyResult> { + let (document, extra_headers, optional_params, timeout) = marshal_inputs( + py, + document, + extra_headers, + optional_params, + timeout_seconds, + )?; + + let result = release_gil(py, || { + pyo3_async_runtimes::tokio::get_runtime().block_on(run_ocr(OcrRequest { + model: &model, + document, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + callbacks: Vec::new(), + guardrails: Vec::new(), + request_metadata: Default::default(), + litellm_call_id: None, + })) + }); + + match result { + Ok(value) => to_py(py, &value), + Err(err) => Err(core_error_to_pyerr(err)), + } +} + +#[pyfunction] +#[pyo3(signature = (model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] +#[allow(clippy::too_many_arguments)] +fn aocr( + py: Python<'_>, + model: String, + document: Py, + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + extra_headers: Option>, + optional_params: Option>, + timeout_seconds: Option, +) -> PyResult> { + let (document, extra_headers, optional_params, timeout) = marshal_inputs( + py, + document, + extra_headers, + optional_params, + timeout_seconds, + )?; + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let value = run_ocr(OcrRequest { + model: &model, + document, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + callbacks: Vec::new(), + guardrails: Vec::new(), + request_metadata: Default::default(), + litellm_call_id: None, + }) + .await + .map_err(core_error_to_pyerr)?; + + Python::attach(|py| to_py(py, &value)) + }) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(ocr, module)?)?; + module.add_function(wrap_pyfunction!(aocr, module)?) +} From d69da842b86358f270d687dd6938f62479b8269c Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 21:01:28 -0700 Subject: [PATCH 55/67] fix(python-bridge): harden sync and async route boundaries --- .github/workflows/test-rust.yml | 5 + litellm-rust/Cargo.lock | 23 +- litellm-rust/Cargo.toml | 2 +- litellm-rust/crates/python-bridge/Cargo.toml | 3 + litellm-rust/crates/python-bridge/src/lib.rs | 115 ++++- .../crates/python-bridge/src/marshal.rs | 7 +- .../src/routes/audio_transcription.rs | 55 +-- .../src/routes/chat_completions.rs | 64 ++- .../python-bridge/src/routes/messages.rs | 74 ++- .../crates/python-bridge/src/routes/mod.rs | 1 + .../crates/python-bridge/src/routes/ocr.rs | 87 ++-- .../python-bridge/src/routes/runtime.rs | 423 ++++++++++++++++++ litellm-rust/crates/python-interop/src/lib.rs | 2 +- .../crates/python-interop/src/marshal.rs | 72 +++ .../rust_bridge/native_route_wheel_test.py | 339 ++++++++++++++ 15 files changed, 1095 insertions(+), 177 deletions(-) create mode 100644 litellm-rust/crates/python-bridge/src/routes/runtime.rs create mode 100644 tests/test_litellm/rust_bridge/native_route_wheel_test.py diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index aada0fcf239..1b71232bc2e 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -9,6 +9,7 @@ on: - "rust-toolchain.toml" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" + - "tests/test_litellm/rust_bridge/native_route_wheel_test.py" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -23,6 +24,7 @@ on: - "rust-toolchain.toml" - ".github/scripts/smoke_test_native_wheel.py" - ".github/scripts/verify_linux_native_wheel.py" + - "tests/test_litellm/rust_bridge/native_route_wheel_test.py" - ".github/workflows/test-rust.yml" permissions: @@ -121,3 +123,6 @@ jobs: env: RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl + + - name: Test native route wheel + run: python tests/test_litellm/rust_bridge/native_route_wheel_test.py dist/*.whl diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index dd41cf0e84b..33f6bb5a87e 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1442,13 +1442,16 @@ name = "litellm-python-bridge" version = "0.1.0" dependencies = [ "criterion", + "futures-util", "litellm-ai-gateway", "litellm-core", "litellm-python-interop", "pyo3", "pyo3-async-runtimes", + "serde", "serde_json", "tokio", + "tokio-tungstenite", ] [[package]] @@ -1669,9 +1672,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "libc", "once_cell", @@ -1697,18 +1700,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -1716,9 +1719,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1728,9 +1731,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index c447d915abe..f99d7b47918 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -18,7 +18,7 @@ litellm-core = { path = "crates/core" } litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false } litellm-python-interop = { path = "crates/python-interop" } axum = "0.7" -pyo3 = "0.29.0" +pyo3 = "0.29.2" pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } pythonize = "0.29.0" rand = "0.8" diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 498003de149..e89f9cd9aa9 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -16,16 +16,19 @@ extension-module = ["pyo3/extension-module"] panic-test = [] [dependencies] +futures-util.workspace = true litellm-core = { workspace = true, features = ["bedrock-auth"] } litellm-ai-gateway = { workspace = true, default-features = false } litellm-python-interop.workspace = true pyo3.workspace = true pyo3-async-runtimes.workspace = true +serde.workspace = true serde_json.workspace = true tokio.workspace = true [dev-dependencies] criterion = "0.8.2" +tokio-tungstenite.workspace = true [[bench]] name = "serialization" diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f628c987220..47df6b905a8 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -6,6 +6,7 @@ mod routes; use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; use pyo3::prelude::*; use pyo3::types::PyAny; +use serde_json::Value; use crate::errors::core_error_to_pyerr; use crate::marshal::{marshal_headers, optional_timeout}; @@ -23,16 +24,16 @@ impl ResponsesWebSocketConnection { _cls: &Bound<'py, pyo3::types::PyType>, py: Python<'py>, url: String, - headers: Option>, + #[pyo3(from_py_with = litellm_python_interop::from_py)] headers: Option, timeout_seconds: Option, ) -> PyResult> { - let headers = marshal_headers(py, headers)?; + let headers = marshal_headers(headers)?; let timeout = optional_timeout(timeout_seconds); pyo3_async_runtimes::tokio::future_into_py(py, async move { let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout) .await .map_err(core_error_to_pyerr)?; - Python::attach(|py| Py::new(py, ResponsesWebSocketConnection { inner })) + Ok(ResponsesWebSocketConnection { inner }) }) } @@ -58,24 +59,36 @@ impl ResponsesWebSocketConnection { } } -#[pymodule] -fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { - errors::register(module)?; - routes::register(module)?; - module.add_class::()?; - diagnostics::register(module) +#[pymodule(gil_used = false)] +mod _native { + use pyo3::prelude::*; + + #[pymodule_init] + fn init(module: &Bound<'_, PyModule>) -> PyResult<()> { + super::errors::register(module)?; + super::routes::register(module)?; + module.add_class::()?; + super::diagnostics::register(module) + } } #[cfg(test)] mod tests { + use std::ffi::CString; + use std::time::Duration; + + use futures_util::{SinkExt, StreamExt}; + use pyo3::types::PyDict; + use tokio::net::TcpListener; + use tokio_tungstenite::{accept_async, tungstenite::Message}; + use super::*; #[test] fn module_registration_preserves_the_public_surface() { Python::initialize(); Python::attach(|py| { - let module = PyModule::new(py, "_native").expect("module should be created"); - _native(&module).expect("module should register"); + let module = pyo3::wrap_pymodule!(_native)(py).into_bound(py); let expected = [ "RustBridgeDeclined", @@ -93,13 +106,79 @@ mod tests { "gil_stats", ]; - for name in expected { - assert!( - module - .hasattr(name) - .expect("attribute lookup should succeed") - ); - } + let public_names: Vec = module + .dict() + .keys() + .extract::>() + .expect("module names should be strings") + .into_iter() + .filter(|name| !name.starts_with("__")) + .collect(); + assert_eq!(public_names, expected); }); } + + #[test] + fn responses_websocket_connection_round_trips_through_python() { + Python::initialize(); + let runtime = pyo3_async_runtimes::tokio::get_runtime(); + let listener = runtime + .block_on(TcpListener::bind("127.0.0.1:0")) + .expect("listener should bind"); + let address = listener + .local_addr() + .expect("listener should have an address"); + let server = runtime.spawn(async move { + let (stream, _) = listener.accept().await.expect("server should accept"); + let mut socket = accept_async(stream) + .await + .expect("handshake should succeed"); + + let message = socket + .next() + .await + .expect("client should send a frame") + .expect("client frame should be valid"); + assert_eq!(message, Message::Text("from-python".into())); + socket + .send(Message::Text("from-server".into())) + .await + .expect("server should reply"); + assert!(matches!(socket.next().await, Some(Ok(Message::Close(_))))); + }); + + Python::attach(|py| { + let module = pyo3::wrap_pymodule!(_native)(py).into_bound(py); + let locals = PyDict::new(py); + locals + .set_item("native", &module) + .expect("module should enter Python locals"); + locals + .set_item("url", format!("ws://{address}")) + .expect("URL should enter Python locals"); + let code = CString::new( + r#" +import asyncio + +async def exercise(): + connection = await native.ResponsesWebSocketConnection.connect(url) + assert type(connection) is native.ResponsesWebSocketConnection + await connection.send_text("from-python") + assert await connection.recv_text() == "from-server" + await connection.close() + assert await connection.recv_text() is None + +asyncio.run(asyncio.wait_for(exercise(), timeout=5)) +"#, + ) + .expect("Python source should not contain null bytes"); + py.run(&code, Some(&locals), Some(&locals)) + .expect("Python WebSocket methods should round trip"); + }); + + runtime + .block_on(async { tokio::time::timeout(Duration::from_secs(5), server).await }) + .expect("server should finish") + .expect("server task should not panic"); + } } diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index 6c070d2a9ee..724514c6bab 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -30,12 +30,9 @@ pub(crate) fn optional_timeout(timeout_seconds: Option) -> Option }) } -pub(crate) fn marshal_headers( - py: Python<'_>, - headers: Option>, -) -> PyResult> { +pub(crate) fn marshal_headers(headers: Option) -> PyResult> { let value = match headers { - Some(headers) => from_py(headers.bind(py))?, + Some(headers) => headers, None => Value::Object(Map::new()), }; let Value::Object(headers) = value else { diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs index 4017cf0887e..a14f3a6baad 100644 --- a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs @@ -1,12 +1,14 @@ use litellm_core::audio_transcription::{ AudioTranscriptionRequest, audio_transcription as run_audio_transcription, }; -use litellm_python_interop::{from_py, release_gil, to_py}; +use litellm_python_interop::from_py; use pyo3::prelude::*; use crate::errors::core_error_to_pyerr; use crate::marshal::{optional_object_to_map, optional_timeout}; +use super::runtime::{run_async, run_sync}; + #[pyfunction] #[pyo3(signature = (model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] #[allow(clippy::too_many_arguments)] @@ -28,9 +30,10 @@ fn transcription( }; let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; let timeout = optional_timeout(timeout_seconds); - let result = release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_audio_transcription( - AudioTranscriptionRequest { + run_sync( + py, + async move { + run_audio_transcription(AudioTranscriptionRequest { model: &model, audio, api_key: api_key.as_deref(), @@ -39,13 +42,11 @@ fn transcription( extra_headers, optional_params, timeout, - }, - )) - }); - match result { - Ok(value) => to_py(py, &value), - Err(err) => Err(core_error_to_pyerr(err)), - } + }) + .await + }, + core_error_to_pyerr, + ) } #[pyfunction] @@ -69,21 +70,23 @@ fn atranscription( }; let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; let timeout = optional_timeout(timeout_seconds); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let value = run_audio_transcription(AudioTranscriptionRequest { - model: &model, - audio, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - }) - .await - .map_err(core_error_to_pyerr)?; - Python::attach(|py| to_py(py, &value)) - }) + run_async( + py, + async move { + run_audio_transcription(AudioTranscriptionRequest { + model: &model, + audio, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + }) + .await + }, + core_error_to_pyerr, + ) } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs index b353765e7cc..b3dcacbe2ac 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs @@ -1,10 +1,10 @@ use std::time::Duration; -use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse}; +use litellm_core::chat_completions::types::ChatCompletionsRequest; use litellm_core::chat_completions::{ chat_completions as run_chat_completions, chat_completions_decline_reason, }; -use litellm_python_interop::{from_py, release_gil, to_py}; +use litellm_python_interop::from_py; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use serde_json::{Map, Value}; @@ -12,12 +12,7 @@ use serde_json::{Map, Value}; use crate::errors::chat_completions_error_to_pyerr; use crate::marshal::{optional_object_to_map, optional_timeout}; -fn chat_completions_response_to_py( - py: Python<'_>, - response: ChatCompletionsResponse, -) -> PyResult> { - to_py(py, &response) -} +use super::runtime::{run_async, run_sync}; type MarshaledChatCompletionsInputs = ( Value, @@ -95,9 +90,10 @@ fn chat_completions( timeout_seconds, )?; - let result = release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_chat_completions( - ChatCompletionsRequest { + run_sync( + py, + async move { + run_chat_completions(ChatCompletionsRequest { model: &model, messages, optional_params, @@ -106,14 +102,11 @@ fn chat_completions( custom_llm_provider: custom_llm_provider.as_deref(), extra_headers, timeout, - }, - )) - }); - - match result { - Ok(response) => chat_completions_response_to_py(py, response), - Err(err) => Err(chat_completions_error_to_pyerr(err)), - } + }) + .await + }, + chat_completions_error_to_pyerr, + ) } #[pyfunction] @@ -138,22 +131,23 @@ fn achat_completions( timeout_seconds, )?; - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let response = run_chat_completions(ChatCompletionsRequest { - model: &model, - messages, - optional_params, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }) - .await - .map_err(chat_completions_error_to_pyerr)?; - - Python::attach(|py| chat_completions_response_to_py(py, response)) - }) + run_async( + py, + async move { + run_chat_completions(ChatCompletionsRequest { + model: &model, + messages, + optional_params, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + }) + .await + }, + chat_completions_error_to_pyerr, + ) } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/litellm-rust/crates/python-bridge/src/routes/messages.rs b/litellm-rust/crates/python-bridge/src/routes/messages.rs index ac361bb0e59..a66bc43dd73 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages.rs @@ -1,8 +1,8 @@ use std::time::Duration; use litellm_core::messages::messages as run_messages; -use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; -use litellm_python_interop::{from_py, release_gil, to_py}; +use litellm_core::messages::types::MessagesRequest; +use litellm_python_interop::from_py; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use serde_json::{Map, Value}; @@ -10,12 +10,7 @@ use serde_json::{Map, Value}; use crate::errors::core_error_to_pyerr; use crate::marshal::{optional_object_to_map, optional_timeout}; -fn messages_response_to_py( - py: Python<'_>, - response: AnthropicMessagesResponse, -) -> PyResult> { - to_py(py, &response) -} +use super::runtime::{run_async, run_sync}; type MarshaledMessagesInputs = (Value, Option>, Option); @@ -52,22 +47,22 @@ fn messages( let (body, extra_headers, timeout) = marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; - let result = release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_messages(MessagesRequest { - model: &model, - body, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - })) - }); - - match result { - Ok(response) => messages_response_to_py(py, response), - Err(err) => Err(core_error_to_pyerr(err)), - } + run_sync( + py, + async move { + run_messages(MessagesRequest { + model: &model, + body, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + }) + .await + }, + core_error_to_pyerr, + ) } #[pyfunction] @@ -86,21 +81,22 @@ fn amessages( let (body, extra_headers, timeout) = marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let response = run_messages(MessagesRequest { - model: &model, - body, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }) - .await - .map_err(core_error_to_pyerr)?; - - Python::attach(|py| messages_response_to_py(py, response)) - }) + run_async( + py, + async move { + run_messages(MessagesRequest { + model: &model, + body, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + }) + .await + }, + core_error_to_pyerr, + ) } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index a2eb8355767..46724a39bc0 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -4,6 +4,7 @@ mod audio_transcription; mod chat_completions; mod messages; mod ocr; +mod runtime; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { ocr::register(module)?; diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr.rs b/litellm-rust/crates/python-bridge/src/routes/ocr.rs index 047bf245d7c..0f6c7577722 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr.rs @@ -1,13 +1,15 @@ use std::time::Duration; use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr}; -use litellm_python_interop::{from_py, release_gil, to_py}; +use litellm_python_interop::from_py; use pyo3::prelude::*; use serde_json::{Map, Value}; use crate::errors::core_error_to_pyerr; use crate::marshal::{optional_object_to_map, optional_timeout}; +use super::runtime::{run_async, run_sync}; + type MarshaledOcrInputs = ( Value, Option>, @@ -55,27 +57,27 @@ fn ocr( timeout_seconds, )?; - let result = release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_ocr(OcrRequest { - model: &model, - document, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - })) - }); - - match result { - Ok(value) => to_py(py, &value), - Err(err) => Err(core_error_to_pyerr(err)), - } + run_sync( + py, + async move { + run_ocr(OcrRequest { + model: &model, + document, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + callbacks: Vec::new(), + guardrails: Vec::new(), + request_metadata: Default::default(), + litellm_call_id: None, + }) + .await + }, + core_error_to_pyerr, + ) } #[pyfunction] @@ -100,26 +102,27 @@ fn aocr( timeout_seconds, )?; - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let value = run_ocr(OcrRequest { - model: &model, - document, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - }) - .await - .map_err(core_error_to_pyerr)?; - - Python::attach(|py| to_py(py, &value)) - }) + run_async( + py, + async move { + run_ocr(OcrRequest { + model: &model, + document, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + callbacks: Vec::new(), + guardrails: Vec::new(), + request_metadata: Default::default(), + litellm_call_id: None, + }) + .await + }, + core_error_to_pyerr, + ) } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/litellm-rust/crates/python-bridge/src/routes/runtime.rs b/litellm-rust/crates/python-bridge/src/routes/runtime.rs new file mode 100644 index 00000000000..87a0c3e0104 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/runtime.rs @@ -0,0 +1,423 @@ +use std::future::Future; +use std::panic::AssertUnwindSafe; +use std::time::Duration; + +use futures_util::FutureExt; +use litellm_core::error::Error; +use litellm_python_interop::{Pythonized, panic_to_pyerr, release_gil}; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::*; +use serde::Serialize; +use tokio::runtime::{Handle, Runtime}; +use tokio::time::{self, MissedTickBehavior}; + +pub(super) fn run_sync( + py: Python<'_>, + future: F, + map_error: fn(Error) -> PyErr, +) -> PyResult> +where + T: Serialize + Send + 'static, + F: Future> + Send + 'static, +{ + run_sync_on( + py, + pyo3_async_runtimes::tokio::get_runtime(), + future, + map_error, + ) +} + +fn run_sync_on( + py: Python<'_>, + runtime: &Runtime, + future: F, + map_error: fn(Error) -> PyErr, +) -> PyResult> +where + T: Serialize + Send + 'static, + F: Future> + Send + 'static, +{ + if Handle::try_current().is_ok() { + return Err(PyRuntimeError::new_err( + "synchronous native routes cannot run from a Tokio context; use the async route", + )); + } + + let result = release_gil(py, move || runtime.block_on(wait_for_sync_result(future)))?; + let result = map_core_result(result, map_error)?; + Pythonized(result).into_pyobject(py).map(Bound::unbind) +} + +pub(super) fn run_async( + py: Python<'_>, + future: F, + map_error: fn(Error) -> PyErr, +) -> PyResult> +where + T: Serialize + Send + 'static, + F: Future> + Send + 'static, +{ + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let result = catch_route_panic(future).await?; + let result = map_core_result(result, map_error)?; + Ok(Pythonized(result)) + }) +} + +fn map_core_result(result: Result, map_error: fn(Error) -> PyErr) -> PyResult { + match result { + Ok(value) => Ok(value), + Err(error) => Err( + std::panic::catch_unwind(AssertUnwindSafe(|| map_error(error))) + .map_err(panic_to_pyerr)?, + ), + } +} + +async fn catch_route_panic(future: F) -> PyResult> +where + F: Future>, +{ + AssertUnwindSafe(future) + .catch_unwind() + .await + .map_err(panic_to_pyerr) +} + +async fn wait_for_sync_result(future: F) -> PyResult> +where + F: Future>, +{ + let future = catch_route_panic(future); + tokio::pin!(future); + + let signal_interval = Duration::from_millis(50); + let mut signal_checks = + time::interval_at(time::Instant::now() + signal_interval, signal_interval); + signal_checks.set_missed_tick_behavior(MissedTickBehavior::Delay); + loop { + tokio::select! { + result = &mut future => return result, + _ = signal_checks.tick() => Python::attach(|py| py.check_signals())?, + } + } +} + +#[cfg(test)] +mod tests { + use std::ffi::CString; + use std::future::poll_fn; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, mpsc}; + use std::task::Poll; + use std::thread; + use std::time::Instant; + + use pyo3::panic::PanicException; + use pyo3::types::{PyDict, PyModule}; + use serde::Serializer; + use tokio::runtime::Builder; + + use super::*; + + fn runtime_error(error: Error) -> PyErr { + PyRuntimeError::new_err(error.to_string()) + } + + fn panicking_error_mapper(_error: Error) -> PyErr { + panic!("error mapper panicked") + } + + struct PanickingOutput; + + static ASYNC_PROBE_COMPLETED: AtomicUsize = AtomicUsize::new(0); + + impl Serialize for PanickingOutput { + fn serialize(&self, _serializer: S) -> Result + where + S: Serializer, + { + panic!("serializer panicked") + } + } + + #[pyfunction] + fn async_serialization_panic(py: Python<'_>) -> PyResult> { + run_async(py, async { Ok(PanickingOutput) }, runtime_error) + } + + #[pyfunction] + fn async_runtime_probe(py: Python<'_>) -> PyResult> { + run_async( + py, + async { + ASYNC_PROBE_COMPLETED.fetch_add(1, Ordering::SeqCst); + Ok(true) + }, + runtime_error, + ) + } + + #[pyfunction] + fn runtime_worker_count() -> usize { + pyo3_async_runtimes::tokio::get_runtime() + .metrics() + .num_workers() + } + + #[pyfunction] + fn runtime_is_responsive(_py: Python<'_>, expected_completions: usize) -> bool { + let completion_deadline = Instant::now() + Duration::from_secs(2); + while ASYNC_PROBE_COMPLETED.load(Ordering::SeqCst) < expected_completions { + if Instant::now() >= completion_deadline { + return false; + } + thread::sleep(Duration::from_millis(1)); + } + + let (heartbeat_tx, heartbeat_rx) = mpsc::sync_channel(1); + pyo3_async_runtimes::tokio::get_runtime().spawn(async move { + let _ = heartbeat_tx.send(()); + }); + heartbeat_rx.recv_timeout(Duration::from_secs(2)).is_ok() + } + + fn extract_bool(py: Python<'_>, result: PyResult>) -> bool { + result + .expect("route should complete") + .bind(py) + .extract() + .expect("result should convert") + } + + #[test] + fn sync_runner_polls_future_on_the_caller_thread() { + Python::initialize(); + Python::attach(|py| { + let caller_thread = std::thread::current().id(); + let result = run_sync( + py, + async move { Ok(std::thread::current().id() == caller_thread) }, + runtime_error, + ); + + assert!(extract_bool(py, result)); + }); + } + + #[test] + fn sync_runner_releases_gil_while_waiting() { + Python::initialize(); + Python::attach(|py| { + let result = run_sync( + py, + async { + let gil_acquired = tokio::time::timeout( + Duration::from_secs(2), + tokio::task::spawn_blocking(|| Python::attach(|_| true)), + ) + .await; + Ok(matches!(gil_acquired, Ok(Ok(true)))) + }, + runtime_error, + ); + + assert!(extract_bool(py, result)); + }); + } + + #[test] + fn sync_runner_rejects_calls_from_a_tokio_context() { + Python::initialize(); + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + + let error = runtime.block_on(async { + Python::attach(|py| { + run_sync::(py, async { Ok(true) }, runtime_error) + .expect_err("sync route should reject a nested Tokio runtime") + }) + }); + + assert_eq!( + error.to_string(), + "RuntimeError: synchronous native routes cannot run from a Tokio context; use the async route" + ); + } + + #[test] + fn sync_runner_can_drive_a_current_thread_runtime() { + Python::initialize(); + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + Python::attach(|py| { + let result = run_sync_on( + py, + &runtime, + async { + tokio::task::yield_now().await; + Ok(true) + }, + runtime_error, + ); + assert!(extract_bool(py, result)); + }); + } + + #[test] + fn sync_runner_maps_a_panicked_future() { + Python::initialize(); + Python::attach(|py| { + let error = run_sync::( + py, + poll_fn(|_| -> Poll> { panic!("route future panicked") }), + runtime_error, + ) + .expect_err("panicked route should become a Python exception"); + + assert!(error.is_instance_of::(py)); + assert_eq!(error.to_string(), "PanicException: route future panicked"); + }); + } + + #[test] + fn sync_runner_maps_a_panicked_error_mapper() { + Python::initialize(); + Python::attach(|py| { + let error = run_sync::( + py, + async { Err(Error::InvalidRequest("invalid".to_string())) }, + panicking_error_mapper, + ) + .expect_err("panicked mapper should become a Python exception"); + + assert!(error.is_instance_of::(py)); + assert_eq!(error.to_string(), "PanicException: error mapper panicked"); + }); + } + + #[test] + fn sync_runner_surfaces_serializer_panics() { + Python::initialize(); + Python::attach(|py| { + let error = run_sync(py, async { Ok(PanickingOutput) }, runtime_error) + .expect_err("serializer panic should become a Python exception"); + + assert!(error.is_instance_of::(py)); + assert_eq!(error.to_string(), "PanicException: serializer panicked"); + }); + } + + #[test] + fn sync_runner_supports_concurrent_callers_on_the_shared_runtime() { + Python::initialize(); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let callers: Vec<_> = (0..2) + .map(|_| { + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + Python::attach(|py| { + extract_bool( + py, + run_sync( + py, + async move { + Ok(tokio::time::timeout(Duration::from_secs(2), barrier.wait()) + .await + .is_ok()) + }, + runtime_error, + ), + ) + }) + }) + }) + .collect(); + let results: Vec<_> = callers + .into_iter() + .map(|caller| caller.join().expect("caller should not panic")) + .collect(); + + assert_eq!(results, vec![true, true]); + } + + #[test] + fn async_runner_surfaces_serializer_panics() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "runtime").expect("module should be created"); + module + .add_function( + wrap_pyfunction!(async_serialization_panic, &module) + .expect("function should wrap"), + ) + .expect("function should register"); + let locals = PyDict::new(py); + locals + .set_item("runtime", &module) + .expect("module should enter Python locals"); + let code = CString::new( + r#" +import asyncio + +async def exercise(): + try: + await runtime.async_serialization_panic() + except BaseException as error: + assert type(error).__name__ == "PanicException" + assert str(error) == "serializer panicked" + else: + raise AssertionError("serializer panic was not raised") + +asyncio.run(exercise()) +"#, + ) + .expect("Python source should not contain null bytes"); + py.run(&code, Some(&locals), Some(&locals)) + .expect("serializer panic should reach the Python awaiter"); + }); + } + + #[test] + fn async_result_delivery_does_not_stall_tokio_workers() { + Python::initialize(); + ASYNC_PROBE_COMPLETED.store(0, Ordering::SeqCst); + Python::attach(|py| { + let module = PyModule::new(py, "runtime").expect("module should be created"); + for function in [ + wrap_pyfunction!(async_runtime_probe, &module).expect("function should wrap"), + wrap_pyfunction!(runtime_worker_count, &module).expect("function should wrap"), + wrap_pyfunction!(runtime_is_responsive, &module).expect("function should wrap"), + ] { + module + .add_function(function) + .expect("function should register"); + } + let locals = PyDict::new(py); + locals + .set_item("runtime", &module) + .expect("module should enter Python locals"); + let code = CString::new( + r#" +import asyncio + +async def exercise(): + worker_count = runtime.runtime_worker_count() + awaitables = [runtime.async_runtime_probe() for _ in range(worker_count)] + assert runtime.runtime_is_responsive(worker_count) + assert await asyncio.gather(*awaitables) == [True] * worker_count + +asyncio.run(exercise()) +"#, + ) + .expect("Python source should not contain null bytes"); + py.run(&code, Some(&locals), Some(&locals)) + .expect("result delivery should leave Tokio workers responsive"); + }); + } +} diff --git a/litellm-rust/crates/python-interop/src/lib.rs b/litellm-rust/crates/python-interop/src/lib.rs index df2bd260fdb..2e562bdae70 100644 --- a/litellm-rust/crates/python-interop/src/lib.rs +++ b/litellm-rust/crates/python-interop/src/lib.rs @@ -2,4 +2,4 @@ mod gil; mod marshal; pub use gil::{release_count, release_gil}; -pub use marshal::{from_py, to_py}; +pub use marshal::{Pythonized, from_py, panic_to_pyerr, to_py}; diff --git a/litellm-rust/crates/python-interop/src/marshal.rs b/litellm-rust/crates/python-interop/src/marshal.rs index c3d0638427c..a16d1e0ae13 100644 --- a/litellm-rust/crates/python-interop/src/marshal.rs +++ b/litellm-rust/crates/python-interop/src/marshal.rs @@ -1,4 +1,8 @@ +use std::any::Any; +use std::panic::{AssertUnwindSafe, catch_unwind}; + use pyo3::exceptions::PyValueError; +use pyo3::panic::PanicException; use pyo3::prelude::*; use serde::Serialize; use serde::de::DeserializeOwned; @@ -18,3 +22,71 @@ where .map(Bound::unbind) .map_err(|error| PyValueError::new_err(error.to_string())) } + +pub struct Pythonized(pub T); + +impl<'py, T> IntoPyObject<'py> for Pythonized +where + T: Serialize, +{ + type Target = PyAny; + type Output = Bound<'py, PyAny>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> PyResult { + catch_unwind(AssertUnwindSafe(|| pythonize::pythonize(py, &self.0))) + .map_err(panic_to_pyerr)? + .map_err(|error| PyValueError::new_err(error.to_string())) + } +} + +pub fn panic_to_pyerr(payload: Box) -> PyErr { + let message = payload + .downcast_ref::() + .map(String::as_str) + .or_else(|| payload.downcast_ref::<&str>().copied()) + .unwrap_or("panic from Rust code"); + PanicException::new_err(message.to_string()) +} + +#[cfg(test)] +mod tests { + use serde::Serializer; + + use super::*; + + struct PanickingSerializer; + + impl Serialize for PanickingSerializer { + fn serialize(&self, _serializer: S) -> Result + where + S: Serializer, + { + panic!("serializer panicked") + } + } + + #[test] + fn pythonized_converts_on_the_attached_thread() { + Python::initialize(); + Python::attach(|py| { + let value: Vec = Pythonized(vec![1, 2, 3]) + .into_pyobject(py) + .and_then(|value| value.extract()) + .expect("value should convert"); + assert_eq!(value, vec![1, 2, 3]); + }); + } + + #[test] + fn pythonized_maps_serializer_panics_to_a_base_exception() { + Python::initialize(); + Python::attach(|py| { + let error = Pythonized(PanickingSerializer) + .into_pyobject(py) + .expect_err("serializer panic should become a Python exception"); + assert!(error.is_instance_of::(py)); + assert_eq!(error.to_string(), "PanicException: serializer panicked"); + }); + } +} diff --git a/tests/test_litellm/rust_bridge/native_route_wheel_test.py b/tests/test_litellm/rust_bridge/native_route_wheel_test.py new file mode 100644 index 00000000000..378194191e5 --- /dev/null +++ b/tests/test_litellm/rust_bridge/native_route_wheel_test.py @@ -0,0 +1,339 @@ +from __future__ import annotations + +import asyncio +import importlib.util +import json +import os +import signal +import subprocess +import sys +import tempfile +import threading +import zipfile +from http.client import HTTPMessage +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from socket import socket as Socket +from typing import Final + +REQUEST_STARTED: Final = threading.Event() +REQUEST_CANCELLED: Final = threading.Event() + +ANTHROPIC_RESPONSE: Final = ( + b'{"id":"msg_native","type":"message","role":"assistant",' + b'"model":"claude-sonnet-4-5","content":[{"type":"text","text":"native-message"}],' + b'"stop_reason":"end_turn","stop_sequence":null,' + b'"usage":{"input_tokens":2,"output_tokens":3}}' +) + + +class NativeRouteHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + content_length: Final = int(self.headers.get("content-length", "0")) + body: Final = json.loads(self.rfile.read(content_length)) + route: Final = self.headers.get("x-test-route") + outcome: Final = self.headers.get("x-test-outcome") + assert_native_request(route, outcome, self.path, self.headers, body) + if outcome == "hang": + REQUEST_STARTED.set() + self.connection.settimeout(5) + if connection_was_cancelled(self.connection): + REQUEST_CANCELLED.set() + return + + status: Final = 429 if outcome == "429" else 200 + response_body: Final = native_response(status, route) + + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(response_body))) + self.send_header("connection", "close") + self.end_headers() + self.wfile.write(response_body) + + def log_message(self, _message_format: str, *_args: object) -> None: + pass + + +def connection_was_cancelled(connection: Socket) -> bool: + try: + return connection.recv(1) == b"" + except TimeoutError: + return False + except OSError: + return True + + +def assert_native_request( + route: str | None, + outcome: str | None, + path: str, + headers: HTTPMessage, + body: object, +) -> None: + if route not in {"ocr", "transcription", "messages", "chat_completions"}: + raise AssertionError(f"unexpected route marker: {route!r}") + if outcome not in {"success", "429", "hang"}: + raise AssertionError(f"unexpected outcome marker: {outcome!r}") + if not isinstance(body, dict): + raise TypeError(f"{route} sent {type(body).__name__}, expected a JSON object") + if route == "ocr": + assert path == "/v1/ocr" + assert headers.get("authorization") == "Bearer sk-native" + assert body["model"] == "mistral-ocr-latest" + assert body["document"]["document_url"] == "https://example.com/document.pdf" + assert body["include_image_base64"] is True + return + if route == "transcription": + assert path == "/model/mistral.voxtral-mini-3b-2507/converse" + assert headers.get("authorization", "").startswith("AWS4-HMAC-SHA256 ") + assert headers.get("x-amz-date") + assert body["messages"][0]["content"][0]["audio"]["source"]["bytes"] == "AQI=" + assert "The audio language is en" in body["messages"][0]["content"][1]["text"] + return + assert path == "/v1/messages" + assert headers.get("x-api-key") == "sk-native" + assert body["model"] == "claude-sonnet-4-5" + if route == "messages": + assert body["max_tokens"] == 16 + assert body["messages"][0]["content"] == "hello-from-messages" + return + assert body["max_tokens"] == 17 + assert body["messages"][0]["content"] == [{"type": "text", "text": "hello-from-chat"}] + + +def native_response(status: int, route: str | None) -> bytes: + if status == 429: + return b'{"error":"native-rate-limit"}' + if route == "ocr": + return b'{"pages":[{"index":0,"markdown":"native-ocr"}]}' + if route == "transcription": + return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}' + return ANTHROPIC_RESPONSE + + +def load_native(native_path: Path) -> object: + module_spec: Final = importlib.util.spec_from_file_location("litellm.rust_bridge._native", native_path) + if module_spec is None or module_spec.loader is None: + raise RuntimeError("cannot create native extension import specification") + native_module: Final = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(native_module) + return native_module + + +def route_kwargs(route: str, api_base: str, outcome: str) -> dict[str, object]: + common: Final = { + "api_base": api_base, + "extra_headers": {"x-test-outcome": outcome, "x-test-route": route}, + "timeout_seconds": 3.0, + } + if route == "ocr": + return common | { + "model": "mistral-ocr-latest", + "document": {"type": "document_url", "document_url": "https://example.com/document.pdf"}, + "api_key": "sk-native", + "custom_llm_provider": "mistral", + "optional_params": {"include_image_base64": True}, + } + if route == "transcription": + return common | { + "model": "mistral.voxtral-mini-3b-2507", + "audio": {"data": "AQI=", "format": "wav", "filename": "audio.wav"}, + "custom_llm_provider": "bedrock", + "optional_params": { + "aws_access_key_id": "native-access-key", + "aws_secret_access_key": "native-secret-key", + "aws_region_name": "us-east-1", + "language": "en", + }, + } + if route == "messages": + return common | { + "model": "claude-sonnet-4-5", + "body": { + "model": "claude-sonnet-4-5", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hello-from-messages"}], + }, + "api_key": "sk-native", + "custom_llm_provider": "anthropic", + } + if route == "chat_completions": + return common | { + "model": "anthropic/claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hello-from-chat"}], + "optional_params": {"max_tokens": 17}, + "api_key": "sk-native", + } + raise AssertionError(f"unknown route: {route}") + + +def assert_success(route: str, response: object) -> None: + if not isinstance(response, dict): + raise TypeError(f"{route} returned {type(response).__name__}, expected dict") + actual: Final = success_value(route, response) + expected: Final = ( + "native-ocr" if route == "ocr" else "native-transcription" if route == "transcription" else "native-message" + ) + if actual != expected: + raise AssertionError(f"{route} returned {actual!r}, expected {expected!r}") + + +def success_value(route: str, response: dict[object, object]) -> object: + if route == "ocr": + return response["pages"][0]["markdown"] + if route == "transcription": + return response["text"] + if route == "messages": + return response["content"][0]["text"] + return response["choices"][0]["message"]["content"] + + +def assert_rate_limit(native: object, route: str, error: BaseException) -> None: + if route == "chat_completions": + upstream_error: Final = native.RustUpstreamError + if not isinstance(error, upstream_error) or error.args[0] != 429: + raise AssertionError(f"{route} returned the wrong 429 error: {error!r}") + return + if not isinstance(error, RuntimeError) or "429" not in str(error): + raise AssertionError(f"{route} returned the wrong 429 error: {error!r}") + + +def exercise_sync(native: object, api_base: str) -> None: + for route in ("ocr", "transcription", "messages", "chat_completions"): + function: Final = getattr(native, route) + assert_success(route, function(**route_kwargs(route, api_base, "success"))) + try: + function(**route_kwargs(route, api_base, "429")) + except (RuntimeError, native.RustUpstreamError) as error: + assert_rate_limit(native, route, error) + else: + raise AssertionError(f"{route} accepted a 429 response") + + +async def exercise_async(native: object, api_base: str) -> None: + for route in ("ocr", "transcription", "messages", "chat_completions"): + function: Final = getattr(native, f"a{route}") + assert_success(route, await function(**route_kwargs(route, api_base, "success"))) + try: + await function(**route_kwargs(route, api_base, "429")) + except (RuntimeError, native.RustUpstreamError) as error: + assert_rate_limit(native, route, error) + else: + raise AssertionError(f"a{route} accepted a 429 response") + + +async def exercise_async_concurrency(native: object, api_base: str) -> None: + responses: Final = await asyncio.wait_for( + asyncio.gather( + *( + native.amessages(**route_kwargs("messages", api_base, "success")) + for _ in range(32) + ) + ), + timeout=15, + ) + for response in responses: + assert_success("messages", response) + + +def exercise_routes(native_path: Path, api_base: str) -> object: + native: Final = load_native(native_path) + exercise_sync(native, api_base) + asyncio.run(exercise_async(native, api_base)) + asyncio.run(exercise_async_concurrency(native, api_base)) + return native + + +def exercise_signal(native: object, api_base: str) -> int: + try: + native.messages( + **route_kwargs("messages", api_base, "hang"), + ) + except KeyboardInterrupt: + sys.stdout.write("KeyboardInterrupt\n") + sys.stdout.flush() + sys.stdin.read(1) + return 0 + raise AssertionError("sync native route ignored SIGINT") + + +def verify_sigint(native_path: Path, api_base: str) -> None: + REQUEST_STARTED.clear() + REQUEST_CANCELLED.clear() + process: Final = subprocess.Popen( + (sys.executable, __file__, "child", str(native_path), api_base), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + if not REQUEST_STARTED.wait(30): + process.kill() + stdout, stderr = process.communicate(timeout=5) + raise AssertionError( + f"native route matrix did not reach the hanging upstream\nstdout:\n{stdout}\nstderr:\n{stderr}" + ) + os.kill(process.pid, signal.SIGINT) + if not REQUEST_CANCELLED.wait(5): + raise AssertionError("interrupted native route did not cancel its upstream future") + if process.poll() is not None: + raise AssertionError("signal child exited before cancellation was observed") + stdout, stderr = process.communicate(input="\n", timeout=5) + if process.returncode != 0 or stdout != "KeyboardInterrupt\n": + raise AssertionError( + f"signal child exited with status {process.returncode}\nstdout:\n{stdout}\nstderr:\n{stderr}" + ) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + + +def verify_wheel(wheel: Path) -> int: + with tempfile.TemporaryDirectory() as temporary_directory, zipfile.ZipFile(wheel) as archive: + wheel_root: Final = Path(temporary_directory) + for member in archive.infolist(): + target: Final = wheel_root / member.filename + if member.is_dir(): + target.mkdir(parents=True, exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(archive.read(member)) + native_members: Final = tuple( + member + for member in archive.infolist() + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + raise AssertionError(f"expected one native extension, found {len(native_members)}") + native_path: Final = wheel_root / native_members[0].filename + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), NativeRouteHandler) + server_thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + api_base: Final = f"http://127.0.0.1:{server.server_address[1]}" + try: + verify_sigint(native_path, api_base) + finally: + server.shutdown() + server.server_close() + server_thread.join(timeout=5) + return 0 + + +def main() -> int: + if len(sys.argv) == 2: + return verify_wheel(Path(sys.argv[1])) + if len(sys.argv) == 4 and sys.argv[1] == "child": + native: Final = exercise_routes(Path(sys.argv[2]), sys.argv[3]) + return exercise_signal(native, sys.argv[3]) + sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) From d549376af39b96020e043c18c93c8319cd0589e4 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 21:06:20 -0700 Subject: [PATCH 56/67] refactor(python-bridge): declare sync and async routes once --- .../crates/python-bridge/src/marshal.rs | 70 ++- .../src/routes/audio_transcription.rs | 144 +++--- .../src/routes/chat_completions.rs | 194 +++----- .../python-bridge/src/routes/definition.rs | 419 ++++++++++++++++++ .../python-bridge/src/routes/messages.rs | 152 +++---- .../crates/python-bridge/src/routes/mod.rs | 5 +- .../crates/python-bridge/src/routes/ocr.rs | 184 +++----- 7 files changed, 728 insertions(+), 440 deletions(-) create mode 100644 litellm-rust/crates/python-bridge/src/routes/definition.rs diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index 724514c6bab..a14e4b55d82 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -1,25 +1,79 @@ use std::collections::HashMap; use std::time::Duration; -use litellm_python_interop::from_py; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use serde_json::{Map, Value}; -pub(crate) fn optional_object_to_map( - py: Python<'_>, +pub(crate) struct RouteOptions { + pub(crate) model: String, + pub(crate) api_key: Option, + pub(crate) api_base: Option, + pub(crate) custom_llm_provider: Option, + pub(crate) extra_headers: Option>, + pub(crate) timeout: Option, +} + +pub(crate) struct RouteOptionsInputs { + pub(crate) model: String, + pub(crate) api_key: Option, + pub(crate) api_base: Option, + pub(crate) custom_llm_provider: Option, + pub(crate) extra_headers: Option, + pub(crate) timeout_seconds: Option, +} + +impl RouteOptions { + pub(crate) fn from_python(inputs: RouteOptionsInputs) -> PyResult { + Ok(Self { + model: inputs.model, + api_key: inputs.api_key, + api_base: inputs.api_base, + custom_llm_provider: inputs.custom_llm_provider, + extra_headers: optional_object("extra_headers", inputs.extra_headers)?, + timeout: optional_timeout(inputs.timeout_seconds), + }) + } +} + +pub(crate) fn required_value( name: &'static str, - value: Option>, + value: Value, + expected: fn(&Value) -> bool, + expected_name: &'static str, +) -> PyResult { + if expected(&value) { + return Ok(value); + } + Err(PyValueError::new_err(format!( + "{name} must be a {expected_name}" + ))) +} + +pub(crate) fn object_or_empty( + name: &'static str, + value: Option, ) -> PyResult> { match value { - Some(value) => match from_py(value.bind(py))? { - Value::Object(map) => Ok(map), - _ => Err(PyValueError::new_err(format!("{name} must be a dict"))), - }, + Some(value) => object(name, value), None => Ok(Map::new()), } } +fn optional_object( + name: &'static str, + value: Option, +) -> PyResult>> { + value.map(|value| object(name, value)).transpose() +} + +fn object(name: &'static str, value: Value) -> PyResult> { + match value { + Value::Object(map) => Ok(map), + _ => Err(PyValueError::new_err(format!("{name} must be a dict"))), + } +} + pub(crate) fn optional_timeout(timeout_seconds: Option) -> Option { timeout_seconds.and_then(|secs| { if secs.is_finite() && secs > 0.0 { diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs index a14f3a6baad..10b86132be7 100644 --- a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs @@ -1,95 +1,71 @@ +use litellm_core::Error; +use std::future::Future; + use litellm_core::audio_transcription::{ AudioTranscriptionRequest, audio_transcription as run_audio_transcription, }; -use litellm_python_interop::from_py; use pyo3::prelude::*; +use serde_json::Value; use crate::errors::core_error_to_pyerr; -use crate::marshal::{optional_object_to_map, optional_timeout}; +use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty}; -use super::runtime::{run_async, run_sync}; +fn prepare_transcription( + inputs: AudioTranscriptionInputs, +) -> PyResult> + Send + 'static> { + let audio = inputs.audio; + let options = RouteOptions::from_python(RouteOptionsInputs { + model: inputs.model, + api_key: inputs.api_key, + api_base: inputs.api_base, + custom_llm_provider: inputs.custom_llm_provider, + extra_headers: inputs.extra_headers, + timeout_seconds: inputs.timeout_seconds, + })?; + let optional_params = object_or_empty("optional_params", inputs.optional_params)?; -#[pyfunction] -#[pyo3(signature = (model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn transcription( - py: Python<'_>, - model: String, - audio: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let audio = from_py(audio.bind(py))?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let timeout = optional_timeout(timeout_seconds); - run_sync( - py, - async move { - run_audio_transcription(AudioTranscriptionRequest { - model: &model, - audio, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - }) - .await - }, - core_error_to_pyerr, - ) + Ok(async move { + let RouteOptions { + model, + api_key, + api_base, + custom_llm_provider, + extra_headers, + timeout, + } = options; + run_audio_transcription(AudioTranscriptionRequest { + model: &model, + audio, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + }) + .await + }) } -#[pyfunction] -#[pyo3(signature = (model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn atranscription( - py: Python<'_>, - model: String, - audio: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let audio = from_py(audio.bind(py))?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let timeout = optional_timeout(timeout_seconds); - run_async( - py, - async move { - run_audio_transcription(AudioTranscriptionRequest { - model: &model, - audio, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - }) - .await - }, - core_error_to_pyerr, - ) -} - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(transcription, module)?)?; - module.add_function(wrap_pyfunction!(atranscription, module)?) +bridge_route! { + sync = transcription, + asynchronous = atranscription, + inputs = AudioTranscriptionInputs, + required = { + model: String, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + audio: Value, + }, + optional = { + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + extra_headers: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + optional_params: Option, + timeout_seconds: Option, + }, + prepare = prepare_transcription, + errors = core_error_to_pyerr, } diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs index b3dcacbe2ac..68b7762cb10 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs @@ -1,64 +1,62 @@ -use std::time::Duration; +use litellm_core::Error; +use std::future::Future; -use litellm_core::chat_completions::types::ChatCompletionsRequest; +use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse}; use litellm_core::chat_completions::{ chat_completions as run_chat_completions, chat_completions_decline_reason, }; -use litellm_python_interop::from_py; -use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use serde_json::{Map, Value}; +use serde_json::Value; use crate::errors::chat_completions_error_to_pyerr; -use crate::marshal::{optional_object_to_map, optional_timeout}; +use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty, required_value}; -use super::runtime::{run_async, run_sync}; +fn prepare_chat_completions( + inputs: ChatCompletionsInputs, +) -> PyResult> + Send + 'static> { + let messages = required_value("messages", inputs.messages, Value::is_array, "list")?; + let optional_params = object_or_empty("optional_params", inputs.optional_params)?; + let options = RouteOptions::from_python(RouteOptionsInputs { + model: inputs.model, + api_key: inputs.api_key, + api_base: inputs.api_base, + custom_llm_provider: inputs.custom_llm_provider, + extra_headers: inputs.extra_headers, + timeout_seconds: inputs.timeout_seconds, + })?; -type MarshaledChatCompletionsInputs = ( - Value, - Map, - Option>, - Option, -); - -fn marshal_chat_completions_inputs( - py: Python<'_>, - messages: Py, - optional_params: Option>, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult { - let messages: Value = from_py(messages.bind(py))?; - if !messages.is_array() { - return Err(PyValueError::new_err("messages must be a list")); - } - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - Ok(( - messages, - optional_params, - extra_headers, - optional_timeout(timeout_seconds), - )) + Ok(async move { + let RouteOptions { + model, + api_key, + api_base, + custom_llm_provider, + extra_headers, + timeout, + } = options; + run_chat_completions(ChatCompletionsRequest { + model: &model, + messages, + optional_params, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + }) + .await + }) } -/// The decline reason for this request, or `None` when the Rust path accepts -/// it. Resolves no credentials and performs no I/O, so a host can ask before -/// committing to either path. #[pyfunction] #[pyo3(signature = (model, messages, optional_params=None, custom_llm_provider=None))] fn chat_completions_decline( - py: Python<'_>, model: String, - messages: Py, - optional_params: Option>, + #[pyo3(from_py_with = litellm_python_interop::from_py)] messages: Value, + #[pyo3(from_py_with = litellm_python_interop::from_py)] optional_params: Option, custom_llm_provider: Option, ) -> PyResult> { - let messages = from_py(messages.bind(py))?; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; + let optional_params = object_or_empty("optional_params", optional_params)?; Ok(chat_completions_decline_reason( &model, custom_llm_provider.as_deref(), @@ -68,90 +66,26 @@ fn chat_completions_decline( .map(str::to_string)) } -#[pyfunction] -#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn chat_completions( - py: Python<'_>, - model: String, - messages: Py, - optional_params: Option>, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (messages, optional_params, extra_headers, timeout) = marshal_chat_completions_inputs( - py, - messages, - optional_params, - extra_headers, - timeout_seconds, - )?; - - run_sync( - py, - async move { - run_chat_completions(ChatCompletionsRequest { - model: &model, - messages, - optional_params, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }) - .await - }, - chat_completions_error_to_pyerr, - ) -} - -#[pyfunction] -#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn achat_completions( - py: Python<'_>, - model: String, - messages: Py, - optional_params: Option>, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (messages, optional_params, extra_headers, timeout) = marshal_chat_completions_inputs( - py, - messages, - optional_params, - extra_headers, - timeout_seconds, - )?; - - run_async( - py, - async move { - run_chat_completions(ChatCompletionsRequest { - model: &model, - messages, - optional_params, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }) - .await - }, - chat_completions_error_to_pyerr, - ) -} - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(chat_completions_decline, module)?)?; - module.add_function(wrap_pyfunction!(chat_completions, module)?)?; - module.add_function(wrap_pyfunction!(achat_completions, module)?) +bridge_route! { + sync = chat_completions, + asynchronous = achat_completions, + inputs = ChatCompletionsInputs, + required = { + model: String, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + messages: Value, + }, + optional = { + #[pyo3(from_py_with = litellm_python_interop::from_py)] + optional_params: Option, + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + extra_headers: Option, + timeout_seconds: Option, + }, + prepare = prepare_chat_completions, + errors = chat_completions_error_to_pyerr, + extra = [chat_completions_decline], } diff --git a/litellm-rust/crates/python-bridge/src/routes/definition.rs b/litellm-rust/crates/python-bridge/src/routes/definition.rs new file mode 100644 index 00000000000..5ba1cc6f1ad --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/definition.rs @@ -0,0 +1,419 @@ +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::*; +use pyo3::types::PyCFunction; + +macro_rules! bridge_route { + ( + sync = $sync_name:ident, + asynchronous = $async_name:ident, + inputs = $inputs:ident, + required = { $($(#[$required_attr:meta])* $required_name:ident: $required_type:ty),+ $(,)? }, + optional = { $($(#[$optional_attr:meta])* $optional_name:ident: $optional_type:ty),* $(,)? }, + prepare = $prepare:path, + errors = $map_error:path + $(, extra = [$($extra:ident),* $(,)?])? + $(,)? + ) => { + struct $inputs { + $($required_name: $required_type,)* + $($optional_name: $optional_type),* + } + + #[pyfunction] + #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))] + #[allow(clippy::too_many_arguments)] + fn $sync_name( + py: pyo3::Python<'_>, + $($(#[$required_attr])* $required_name: $required_type,)* + $($(#[$optional_attr])* $optional_name: $optional_type),* + ) -> pyo3::PyResult> { + let future = $prepare($inputs { + $($required_name,)* + $($optional_name),* + })?; + $crate::routes::runtime::run_sync(py, future, $map_error) + } + + #[pyfunction] + #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))] + #[allow(clippy::too_many_arguments)] + fn $async_name( + py: pyo3::Python<'_>, + $($(#[$required_attr])* $required_name: $required_type,)* + $($(#[$optional_attr])* $optional_name: $optional_type),* + ) -> pyo3::PyResult> { + let future = $prepare($inputs { + $($required_name,)* + $($optional_name),* + })?; + $crate::routes::runtime::run_async(py, future, $map_error) + } + + pub(super) fn register( + module: &pyo3::Bound<'_, pyo3::types::PyModule>, + ) -> pyo3::PyResult<()> { + $($($crate::routes::definition::add_function(module, pyo3::wrap_pyfunction!($extra, module)?)?;)*)? + $crate::routes::definition::add_function(module, pyo3::wrap_pyfunction!($sync_name, module)?)?; + $crate::routes::definition::add_function(module, pyo3::wrap_pyfunction!($async_name, module)?)?; + Ok(()) + } + }; +} + +pub(super) fn add_function( + module: &Bound<'_, PyModule>, + function: Bound<'_, PyCFunction>, +) -> PyResult<()> { + let name: String = function.getattr("__name__")?.extract()?; + if module.hasattr(&name)? { + return Err(PyRuntimeError::new_err(format!( + "duplicate native route: {name}" + ))); + } + module.add_function(function) +} + +#[cfg(test)] +mod tests { + use std::ffi::CString; + use std::sync::atomic::{AtomicBool, Ordering}; + + use litellm_core::error::Error; + use pyo3::exceptions::PyLookupError; + use pyo3::types::{PyDict, PyList}; + + use super::*; + + mod synthetic { + use std::future::{Future, pending}; + + use super::*; + + static FUTURE_DROPPED: AtomicBool = AtomicBool::new(false); + + struct DropGuard; + + impl Drop for DropGuard { + fn drop(&mut self) { + FUTURE_DROPPED.store(true, Ordering::SeqCst); + } + } + + #[pyfunction] + fn future_dropped() -> bool { + FUTURE_DROPPED.load(Ordering::SeqCst) + } + + bridge_route! { + sync = echo, + asynchronous = aecho, + inputs = EchoInputs, + required = { value: String }, + optional = {}, + prepare = prepare_echo, + errors = map_error, + extra = [future_dropped], + } + + fn prepare_echo( + inputs: EchoInputs, + ) -> PyResult> + Send + 'static> { + FUTURE_DROPPED.store(false, Ordering::SeqCst); + let drop_guard = (inputs.value == "pending").then_some(DropGuard); + Ok(async move { + let _drop_guard = drop_guard; + tokio::task::yield_now().await; + match inputs.value.as_str() { + "error" => Err(Error::InvalidRequest("synthetic error".to_string())), + "map_panic" => Err(Error::InvalidRequest("panic in mapper".to_string())), + "panic" => panic!("synthetic panic"), + "pending" => { + pending::<()>().await; + unreachable!() + } + _ => Ok(inputs.value), + } + }) + } + + fn map_error(error: Error) -> PyErr { + if matches!(&error, Error::InvalidRequest(message) if message == "panic in mapper") { + panic!("synthetic mapper panic") + } + PyLookupError::new_err(error.to_string()) + } + } + + #[test] + fn sync_and_async_route_signatures_match_the_python_contract() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "routes").expect("module should be created"); + crate::routes::register(&module).expect("routes should register"); + let routes = [ + ( + "ocr", + "aocr", + "(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)", + ), + ( + "transcription", + "atranscription", + "(model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)", + ), + ( + "messages", + "amessages", + "(model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None)", + ), + ( + "chat_completions", + "achat_completions", + "(model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None)", + ), + ]; + + for (sync_name, async_name, expected) in routes { + let sync_signature: String = module + .getattr(sync_name) + .and_then(|function| function.getattr("__text_signature__")) + .and_then(|signature| signature.extract()) + .expect("sync signature should be available"); + let async_signature: String = module + .getattr(async_name) + .and_then(|function| function.getattr("__text_signature__")) + .and_then(|signature| signature.extract()) + .expect("async signature should be available"); + + assert_eq!(sync_signature, expected); + assert_eq!(async_signature, expected); + } + }); + } + + #[test] + fn sync_and_async_routes_apply_the_same_input_validation() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "routes").expect("module should be created"); + crate::routes::register(&module).expect("routes should register"); + + let invalid_messages = PyDict::new(py); + let sync_chat_error = module + .getattr("chat_completions") + .and_then(|function| function.call1(("model", &invalid_messages))) + .expect_err("sync chat should reject a non-list messages value"); + let async_chat_error = module + .getattr("achat_completions") + .and_then(|function| function.call1(("model", &invalid_messages))) + .expect_err("async chat should reject a non-list messages value"); + + assert_eq!( + sync_chat_error.to_string(), + "ValueError: messages must be a list" + ); + assert_eq!(async_chat_error.to_string(), sync_chat_error.to_string()); + + let invalid_body = PyList::empty(py); + let sync_messages_error = module + .getattr("messages") + .and_then(|function| function.call1(("model", &invalid_body))) + .expect_err("sync Messages should reject a non-dict body"); + let async_messages_error = module + .getattr("amessages") + .and_then(|function| function.call1(("model", &invalid_body))) + .expect_err("async Messages should reject a non-dict body"); + + assert_eq!( + sync_messages_error.to_string(), + "ValueError: body must be a dict" + ); + assert_eq!( + async_messages_error.to_string(), + sync_messages_error.to_string() + ); + + let invalid_headers = PyList::empty(py); + let kwargs = PyDict::new(py); + kwargs + .set_item("extra_headers", &invalid_headers) + .expect("kwargs should accept extra_headers"); + let document = PyDict::new(py); + + for (sync_name, async_name) in [("ocr", "aocr"), ("transcription", "atranscription")] { + let sync_error = module + .getattr(sync_name) + .and_then(|function| function.call(("model", &document), Some(&kwargs))) + .expect_err("sync route should reject non-dict extra_headers"); + let async_error = module + .getattr(async_name) + .and_then(|function| function.call(("model", &document), Some(&kwargs))) + .expect_err("async route should reject non-dict extra_headers"); + + assert_eq!( + sync_error.to_string(), + "ValueError: extra_headers must be a dict" + ); + assert_eq!(async_error.to_string(), sync_error.to_string()); + } + }); + } + + #[test] + fn route_input_validation_preserves_left_to_right_order() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "routes").expect("module should be created"); + crate::routes::register(&module).expect("routes should register"); + let invalid = PyList::empty(py); + + let chat_kwargs = PyDict::new(py); + chat_kwargs + .set_item("optional_params", &invalid) + .expect("kwargs should accept optional_params"); + chat_kwargs + .set_item("extra_headers", &invalid) + .expect("kwargs should accept extra_headers"); + let invalid_messages = PyDict::new(py); + let error = module + .getattr("chat_completions") + .and_then(|function| { + function.call(("model", &invalid_messages), Some(&chat_kwargs)) + }) + .expect_err("messages should be validated first"); + assert_eq!(error.to_string(), "ValueError: messages must be a list"); + + let valid_messages = PyList::empty(py); + let error = module + .getattr("chat_completions") + .and_then(|function| function.call(("model", &valid_messages), Some(&chat_kwargs))) + .expect_err("optional_params should be validated before headers"); + assert_eq!( + error.to_string(), + "ValueError: optional_params must be a dict" + ); + + let headers_kwargs = PyDict::new(py); + headers_kwargs + .set_item("extra_headers", &invalid) + .expect("kwargs should accept extra_headers"); + let invalid_body = PyList::empty(py); + let error = module + .getattr("messages") + .and_then(|function| function.call(("model", &invalid_body), Some(&headers_kwargs))) + .expect_err("body should be validated before headers"); + assert_eq!(error.to_string(), "ValueError: body must be a dict"); + + let invalid_payload = + PyModule::new(py, "invalid_payload").expect("invalid payload should be created"); + for name in ["ocr", "transcription"] { + let error = module + .getattr(name) + .and_then(|function| { + function.call(("model", &invalid_payload), Some(&headers_kwargs)) + }) + .expect_err("payload should be validated before headers"); + assert!(!error.to_string().contains("extra_headers")); + } + }); + } + + #[test] + fn generated_routes_execute_sync_and_async_contracts() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "synthetic").expect("module should be created"); + synthetic::register(&module).expect("routes should register"); + + let sync_value: String = module + .getattr("echo") + .and_then(|function| function.call1(("sync",))) + .and_then(|value| value.extract()) + .expect("sync route should return its value"); + assert_eq!(sync_value, "sync"); + + let sync_error = module + .getattr("echo") + .and_then(|function| function.call1(("error",))) + .expect_err("sync route should map its error"); + assert!(sync_error.is_instance_of::(py)); + assert_eq!( + sync_error.to_string(), + "LookupError: invalid request: synthetic error" + ); + + let locals = PyDict::new(py); + locals + .set_item("routes", &module) + .expect("module should enter Python locals"); + let code = CString::new( + r#" +import asyncio + +async def exercise(): + assert await routes.aecho("async") == "async" + + try: + await routes.aecho("error") + except LookupError as error: + assert str(error) == "invalid request: synthetic error" + else: + raise AssertionError("mapped error was not raised") + + try: + await routes.aecho("panic") + except BaseException as error: + assert type(error).__name__ == "PanicException" + assert str(error) == "synthetic panic" + else: + raise AssertionError("panic was not raised") + + try: + await routes.aecho("map_panic") + except BaseException as error: + assert type(error).__name__ == "PanicException" + assert str(error) == "synthetic mapper panic" + else: + raise AssertionError("mapper panic was not raised") + + task = asyncio.ensure_future(routes.aecho("pending")) + await asyncio.sleep(0) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + else: + raise AssertionError("cancelled route completed") + + for _ in range(100): + if routes.future_dropped(): + break + await asyncio.sleep(0.001) + assert routes.future_dropped() + +asyncio.run(exercise()) +"#, + ) + .expect("Python source should not contain null bytes"); + py.run(&code, Some(&locals), Some(&locals)) + .expect("async route contract should hold"); + }); + } + + #[test] + fn route_registration_rejects_duplicate_python_names() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "synthetic").expect("module should be created"); + synthetic::register(&module).expect("first registration should succeed"); + let error = synthetic::register(&module) + .expect_err("duplicate registration should be rejected"); + + assert_eq!( + error.to_string(), + "RuntimeError: duplicate native route: future_dropped" + ); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages.rs b/litellm-rust/crates/python-bridge/src/routes/messages.rs index a66bc43dd73..2bb64a7a763 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages.rs @@ -1,105 +1,65 @@ -use std::time::Duration; - +use litellm_core::Error; use litellm_core::messages::messages as run_messages; -use litellm_core::messages::types::MessagesRequest; -use litellm_python_interop::from_py; -use pyo3::exceptions::PyValueError; +use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; use pyo3::prelude::*; -use serde_json::{Map, Value}; +use serde_json::Value; +use std::future::Future; use crate::errors::core_error_to_pyerr; -use crate::marshal::{optional_object_to_map, optional_timeout}; +use crate::marshal::{RouteOptions, RouteOptionsInputs, required_value}; -use super::runtime::{run_async, run_sync}; +fn prepare_messages( + inputs: MessagesInputs, +) -> PyResult> + Send + 'static> { + let body = required_value("body", inputs.body, Value::is_object, "dict")?; + let options = RouteOptions::from_python(RouteOptionsInputs { + model: inputs.model, + api_key: inputs.api_key, + api_base: inputs.api_base, + custom_llm_provider: inputs.custom_llm_provider, + extra_headers: inputs.extra_headers, + timeout_seconds: inputs.timeout_seconds, + })?; -type MarshaledMessagesInputs = (Value, Option>, Option); - -fn marshal_messages_inputs( - py: Python<'_>, - body: Py, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult { - let body: Value = from_py(body.bind(py))?; - if !body.is_object() { - return Err(PyValueError::new_err("body must be a dict")); - } - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - Ok((body, extra_headers, optional_timeout(timeout_seconds))) + Ok(async move { + let RouteOptions { + model, + api_key, + api_base, + custom_llm_provider, + extra_headers, + timeout, + } = options; + run_messages(MessagesRequest { + model: &model, + body, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + }) + .await + }) } -#[pyfunction] -#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn messages( - py: Python<'_>, - model: String, - body: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (body, extra_headers, timeout) = - marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; - - run_sync( - py, - async move { - run_messages(MessagesRequest { - model: &model, - body, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }) - .await - }, - core_error_to_pyerr, - ) -} - -#[pyfunction] -#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn amessages( - py: Python<'_>, - model: String, - body: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (body, extra_headers, timeout) = - marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; - - run_async( - py, - async move { - run_messages(MessagesRequest { - model: &model, - body, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }) - .await - }, - core_error_to_pyerr, - ) -} - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(messages, module)?)?; - module.add_function(wrap_pyfunction!(amessages, module)?) +bridge_route! { + sync = messages, + asynchronous = amessages, + inputs = MessagesInputs, + required = { + model: String, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + body: Value, + }, + optional = { + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + extra_headers: Option, + timeout_seconds: Option, + }, + prepare = prepare_messages, + errors = core_error_to_pyerr, } diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index 46724a39bc0..3590ea3a769 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -1,10 +1,13 @@ use pyo3::prelude::*; +#[macro_use] +mod definition; +mod runtime; + mod audio_transcription; mod chat_completions; mod messages; mod ocr; -mod runtime; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { ocr::register(module)?; diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr.rs b/litellm-rust/crates/python-bridge/src/routes/ocr.rs index 0f6c7577722..5588c400972 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr.rs @@ -1,131 +1,73 @@ -use std::time::Duration; +use litellm_core::Error; +use std::future::Future; use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr}; -use litellm_python_interop::from_py; use pyo3::prelude::*; -use serde_json::{Map, Value}; +use serde_json::Value; use crate::errors::core_error_to_pyerr; -use crate::marshal::{optional_object_to_map, optional_timeout}; +use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty}; -use super::runtime::{run_async, run_sync}; +fn prepare_ocr( + inputs: OcrInputs, +) -> PyResult> + Send + 'static> { + let document = inputs.document; + let options = RouteOptions::from_python(RouteOptionsInputs { + model: inputs.model, + api_key: inputs.api_key, + api_base: inputs.api_base, + custom_llm_provider: inputs.custom_llm_provider, + extra_headers: inputs.extra_headers, + timeout_seconds: inputs.timeout_seconds, + })?; + let optional_params = object_or_empty("optional_params", inputs.optional_params)?; -type MarshaledOcrInputs = ( - Value, - Option>, - Map, - Option, -); - -fn marshal_inputs( - py: Python<'_>, - document: Py, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult { - let document = from_py(document.bind(py))?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let timeout = optional_timeout(timeout_seconds); - - Ok((document, extra_headers, optional_params, timeout)) + Ok(async move { + let RouteOptions { + model, + api_key, + api_base, + custom_llm_provider, + extra_headers, + timeout, + } = options; + run_ocr(OcrRequest { + model: &model, + document, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + callbacks: Vec::new(), + guardrails: Vec::new(), + request_metadata: Default::default(), + litellm_call_id: None, + }) + .await + }) } -#[pyfunction] -#[pyo3(signature = (model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn ocr( - py: Python<'_>, - model: String, - document: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (document, extra_headers, optional_params, timeout) = marshal_inputs( - py, - document, - extra_headers, - optional_params, - timeout_seconds, - )?; - - run_sync( - py, - async move { - run_ocr(OcrRequest { - model: &model, - document, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - }) - .await - }, - core_error_to_pyerr, - ) -} - -#[pyfunction] -#[pyo3(signature = (model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn aocr( - py: Python<'_>, - model: String, - document: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (document, extra_headers, optional_params, timeout) = marshal_inputs( - py, - document, - extra_headers, - optional_params, - timeout_seconds, - )?; - - run_async( - py, - async move { - run_ocr(OcrRequest { - model: &model, - document, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - }) - .await - }, - core_error_to_pyerr, - ) -} - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(ocr, module)?)?; - module.add_function(wrap_pyfunction!(aocr, module)?) +bridge_route! { + sync = ocr, + asynchronous = aocr, + inputs = OcrInputs, + required = { + model: String, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + document: Value, + }, + optional = { + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + extra_headers: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + optional_params: Option, + timeout_seconds: Option, + }, + prepare = prepare_ocr, + errors = core_error_to_pyerr, } From f2ee0a46f4aa77da504017ff1570b1fa611049b5 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 10:54:42 -0700 Subject: [PATCH 57/67] feat(python): unify Rust opt-in and bridge policy --- litellm/__init__.py | 2 +- litellm/llms/custom_httpx/llm_http_handler.py | 17 +- litellm/ocr/main.py | 10 +- litellm/rust_bridge/__init__.py | 2 +- litellm/rust_bridge/bindings.py | 47 +++++ litellm/rust_bridge/chat_completions.py | 12 +- litellm/rust_bridge/configuration.py | 157 +++++++++++++++ litellm/rust_bridge/ocr.py | 61 +----- litellm/rust_bridge/runtime.py | 180 ++++++++++++++++++ litellm/types/router.py | 1 + .../test_rust_bridge_messages.py | 36 ++-- tests/test_litellm/ocr/test_rust_bridge.py | 35 ++-- .../responses/test_rust_bridge_websocket.py | 23 ++- .../test_litellm/rust_bridge/test_bindings.py | 21 ++ .../rust_bridge/test_chat_completions.py | 83 ++++---- .../rust_bridge/test_configuration.py | 151 +++++++++++++++ .../test_litellm/rust_bridge/test_runtime.py | 95 +++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 18 files changed, 777 insertions(+), 160 deletions(-) create mode 100644 litellm/rust_bridge/bindings.py create mode 100644 litellm/rust_bridge/configuration.py create mode 100644 litellm/rust_bridge/runtime.py create mode 100644 tests/test_litellm/rust_bridge/test_bindings.py create mode 100644 tests/test_litellm/rust_bridge/test_configuration.py create mode 100644 tests/test_litellm/rust_bridge/test_runtime.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 4eeececdb7e..f35fb26abf6 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1416,7 +1416,7 @@ from .skills.main import ( ) from .containers.main import * from .ocr.main import * -from .rust_bridge.ocr import use_litellm_rust +from .rust_bridge import use_litellm_rust from .rag.main import * from .sandbox.main import * from .search.main import * diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 6f42d42de00..3cd6ee54069 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1,6 +1,5 @@ import asyncio import json -import os import ssl from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from contextlib import asynccontextmanager @@ -159,7 +158,11 @@ def _rust_responses_websocket_enabled( custom_llm_provider: str | None, litellm_params: GenericLiteLLMParams, ) -> bool: - return custom_llm_provider == "openai" and litellm_params.get("rust") is True + from litellm.rust_bridge.configuration import rust_enabled + + raw_request_override: Final = litellm_params.get("rust") + request_override: Final = raw_request_override if isinstance(raw_request_override, bool) else None + return custom_llm_provider == "openai" and rust_enabled(request_override=request_override) from .http_handler import get_shared_realtime_ssl_context @@ -2364,10 +2367,6 @@ class BaseLLMHTTPHandler: "anthropic_messages", ) - @staticmethod - def _rust_env_enabled() -> bool: - return os.getenv("LITELLM_RUST", "").strip().lower() in {"1", "true", "yes", "on"} - @staticmethod async def _maybe_rust_anthropic_messages( *, @@ -2383,7 +2382,11 @@ class BaseLLMHTTPHandler: ) -> AnthropicMessagesResponse | None: if custom_llm_provider not in ("azure_ai", "anthropic"): return None - if litellm_params.get("rust") is not True and not BaseLLMHTTPHandler._rust_env_enabled(): + from litellm.rust_bridge.configuration import rust_enabled + + raw_request_override: Final = litellm_params.get("rust") + request_override: Final = raw_request_override if isinstance(raw_request_override, bool) else None + if not rust_enabled(request_override=request_override): return None if has_agentic_hook: return None diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index b918f013700..b260ec6e06f 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -194,6 +194,12 @@ def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool: return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS +def _rust_ocr_enabled(prepared_request: _PreparedOCRRequest) -> bool: + raw_request_override: Final = prepared_request.litellm_params.get("rust") + request_override: Final = raw_request_override if isinstance(raw_request_override, bool) else None + return rust_ocr_bridge.rust_ocr_enabled(request_override=request_override) + + def _rust_bridge_optional_params( prepared_request: _PreparedOCRRequest, resolve_secret: Callable[[str], str | None], @@ -422,7 +428,7 @@ async def aocr( custom_llm_provider = prepared.custom_llm_provider completion_kwargs.update({"model": model, "custom_llm_provider": custom_llm_provider}) - if _rust_ocr_supported(prepared) and rust_ocr_bridge.rust_ocr_enabled(): + if _rust_ocr_supported(prepared) and _rust_ocr_enabled(prepared): from litellm.secret_managers.main import get_secret_str rust_response: Final = await _run_rust_aocr( @@ -694,7 +700,7 @@ def ocr( custom_llm_provider = prepared.custom_llm_provider completion_kwargs.update({"model": model, "custom_llm_provider": custom_llm_provider}) - if _rust_ocr_supported(prepared) and rust_ocr_bridge.rust_ocr_enabled(): + if _rust_ocr_supported(prepared) and _rust_ocr_enabled(prepared): from litellm.secret_managers.main import get_secret_str rust_response: Final = _run_rust_ocr( diff --git a/litellm/rust_bridge/__init__.py b/litellm/rust_bridge/__init__.py index 3da5b98449b..e6d8ffef48c 100644 --- a/litellm/rust_bridge/__init__.py +++ b/litellm/rust_bridge/__init__.py @@ -1,9 +1,9 @@ """LiteLLM Rust bridge package.""" +from litellm.rust_bridge.configuration import use_litellm_rust from litellm.rust_bridge.loader import ( get_native_bridge, native_bridge_available, ) -from litellm.rust_bridge.ocr import use_litellm_rust __all__ = ["get_native_bridge", "native_bridge_available", "use_litellm_rust"] diff --git a/litellm/rust_bridge/bindings.py b/litellm/rust_bridge/bindings.py new file mode 100644 index 00000000000..a4c529f7deb --- /dev/null +++ b/litellm/rust_bridge/bindings.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import Final, Generic, TypeVar, cast + +from litellm.rust_bridge.loader import get_native_bridge + +BindingT = TypeVar("BindingT") + + +class _Unset: + pass + + +_UNSET: Final = _Unset() + + +class NativeBinding(Generic[BindingT]): + """Resolve one native attribute with an explicit, resettable test override.""" + + def __init__(self, attribute: str) -> None: + self._attribute: Final = attribute + self._override: BindingT | None | _Unset = _UNSET + + def load(self) -> BindingT | None: + if not isinstance(self._override, _Unset): + return self._override + native: Final = get_native_bridge() + if native is None: + return None + return cast(BindingT | None, getattr(native, self._attribute, None)) + + def override(self, value: BindingT | None) -> None: + self._override = value + + def reset(self) -> None: + self._override = _UNSET + + +def native_exception_types() -> tuple[type[BaseException], type[BaseException]] | None: + native: Final = get_native_bridge() + if native is None: + return None + declined: Final = getattr(native, "RustBridgeDeclined", None) + upstream: Final = getattr(native, "RustUpstreamError", None) + if not isinstance(declined, type) or not isinstance(upstream, type): + return None + return declined, upstream diff --git a/litellm/rust_bridge/chat_completions.py b/litellm/rust_bridge/chat_completions.py index acda3086051..c599667ab17 100644 --- a/litellm/rust_bridge/chat_completions.py +++ b/litellm/rust_bridge/chat_completions.py @@ -13,7 +13,6 @@ retrying it there would bill the customer for the same work twice. from __future__ import annotations import json -import os from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Protocol @@ -27,6 +26,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo convert_to_model_response_object, ) from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned +from litellm.rust_bridge.configuration import rust_enabled from litellm.rust_bridge.loader import get_native_bridge from litellm.rust_bridge.timeouts import timeout_to_seconds from litellm.types.utils import ModelResponse @@ -44,8 +44,6 @@ _LITELLM_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) RUST_RESPONSE_HEADER: Final = "x-litellm-rust" -_TRUTHY_ENV_VALUES: Final = frozenset({"1", "true", "yes", "on"}) - class RustChatCompletions(Protocol): def __call__( @@ -181,10 +179,6 @@ def load_rust_achat_completions() -> RustAchatCompletions | None: return loaded -def _env_enables_rust() -> bool: - return os.getenv("LITELLM_RUST", "").strip().lower() in _TRUTHY_ENV_VALUES - - def _load_rust_decline() -> RustChatCompletionsDecline | None: if _STATE.decline is not None: return _STATE.decline @@ -253,8 +247,8 @@ def rust_chat_completions_accepts( return False if stream: return False - opted_in: Final = litellm_params is not None and litellm_params.get("rust") is True - if not opted_in and not _env_enables_rust(): + request_override: Final = litellm_params.get("rust") if litellm_params is not None else None + if not rust_enabled(request_override=request_override if isinstance(request_override, bool) else None): return False if _litellm_metadata_reaches_the_provider(custom_llm_provider, litellm_params): verbose_logger.debug("Rust chat completions declined (litellm metadata user_id); using the Python path") diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py new file mode 100644 index 00000000000..a8eecd022d6 --- /dev/null +++ b/litellm/rust_bridge/configuration.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import os +import warnings +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from litellm.rust_bridge.messages import RustAmessages, RustMessages + from litellm.rust_bridge.ocr import RustAocr, RustOcr + from litellm.rust_bridge.responses_websocket import RustResponsesWebSocketConnection + from litellm.rust_bridge.transcription import RustAtranscription, RustTranscription + +DEFAULT_RUST_ENABLED: Final = False +_TRUE_ENV_VALUES: Final = frozenset({"1", "true", "yes", "on"}) +_FALSE_ENV_VALUES: Final = frozenset({"0", "false", "no", "off"}) +_GLOBAL_ENV_NAME: Final = "LITELLM_RUST" +_LEGACY_OCR_ENV_NAME: Final = "LITELLM_USE_RUST_OCR" + + +class _Unset: + pass + + +_UNSET: Final = _Unset() + + +class _RustConfiguration: + def __init__(self) -> None: + self.override: bool | None = None + + +_CONFIGURATION: Final = _RustConfiguration() + + +def _parse_env_bool(name: str, value: str | None) -> bool | None: + if value is None: + return None + normalized: Final = value.strip().lower() + if normalized in _TRUE_ENV_VALUES: + return True + if normalized in _FALSE_ENV_VALUES: + return False + accepted: Final = ", ".join(sorted(_TRUE_ENV_VALUES | _FALSE_ENV_VALUES)) + raise ValueError(f"{name} must be one of: {accepted}") + + +def resolve_rust_enabled( + *, + request_override: bool | None, + process_override: bool | None, + environment_override: bool | None, + legacy_ocr_override: bool | None = None, + release_default: bool = DEFAULT_RUST_ENABLED, +) -> bool: + if request_override is not None: + return request_override + if process_override is not None: + return process_override + if environment_override is not None: + return environment_override + if legacy_ocr_override is not None: + return legacy_ocr_override + return release_default + + +def rust_enabled(*, request_override: bool | None = None) -> bool: + if request_override is not None: + return request_override + process_override: Final = _CONFIGURATION.override + if process_override is not None: + return process_override + return resolve_rust_enabled( + request_override=None, + process_override=None, + environment_override=_parse_env_bool(_GLOBAL_ENV_NAME, os.getenv(_GLOBAL_ENV_NAME)), + ) + + +def rust_ocr_enabled(*, request_override: bool | None = None) -> bool: + if request_override is not None: + return request_override + process_override: Final = _CONFIGURATION.override + if process_override is not None: + return process_override + global_override: Final = _parse_env_bool(_GLOBAL_ENV_NAME, os.getenv(_GLOBAL_ENV_NAME)) + legacy_override: Final = ( + None if global_override is not None else _parse_env_bool(_LEGACY_OCR_ENV_NAME, os.getenv(_LEGACY_OCR_ENV_NAME)) + ) + if legacy_override is not None: + warnings.warn( + f"{_LEGACY_OCR_ENV_NAME} is deprecated; use {_GLOBAL_ENV_NAME} instead", + DeprecationWarning, + stacklevel=2, + ) + return resolve_rust_enabled( + request_override=None, + process_override=None, + environment_override=global_override, + legacy_ocr_override=legacy_override, + ) + + +def reset_rust_configuration() -> None: + _CONFIGURATION.override = None + + +def use_litellm_rust( + enabled: bool = True, + *, + ocr: RustOcr | None | _Unset = _UNSET, + aocr: RustAocr | None | _Unset = _UNSET, + messages: RustMessages | None | _Unset = _UNSET, + amessages: RustAmessages | None | _Unset = _UNSET, + responses_websocket: type[RustResponsesWebSocketConnection] | None | _Unset = _UNSET, + transcription: RustTranscription | None | _Unset = _UNSET, + atranscription: RustAtranscription | None | _Unset = _UNSET, +) -> None: + """Set the process override for optional Rust paths. + + Rust-only paths, including Bedrock transcription, are not controlled by this switch. + """ + _CONFIGURATION.override = enabled + bindings: Final = (ocr, aocr, messages, amessages, responses_websocket, transcription, atranscription) + if all(isinstance(binding, _Unset) for binding in bindings): + return + warnings.warn( + "Injecting Rust bridge implementations through use_litellm_rust() is deprecated; " + "use the internal bridge setters in tests", + DeprecationWarning, + stacklevel=2, + ) + + if not isinstance(ocr, _Unset) or not isinstance(aocr, _Unset): + from litellm.rust_bridge.ocr import set_rust_ocr + + if not isinstance(ocr, _Unset): + set_rust_ocr(ocr=ocr) + if not isinstance(aocr, _Unset): + set_rust_ocr(aocr=aocr) + if not isinstance(messages, _Unset) or not isinstance(amessages, _Unset): + from litellm.rust_bridge.messages import set_rust_messages + + if not isinstance(messages, _Unset): + set_rust_messages(messages=messages) + if not isinstance(amessages, _Unset): + set_rust_messages(amessages=amessages) + if not isinstance(responses_websocket, _Unset): + from litellm.rust_bridge.responses_websocket import set_rust_responses_websocket + + set_rust_responses_websocket(connection=responses_websocket) + if not isinstance(transcription, _Unset) or not isinstance(atranscription, _Unset): + from litellm.rust_bridge.transcription import configure_rust_transcription + + if not isinstance(transcription, _Unset): + configure_rust_transcription(transcription=transcription) + if not isinstance(atranscription, _Unset): + configure_rust_transcription(atranscription=atranscription) diff --git a/litellm/rust_bridge/ocr.py b/litellm/rust_bridge/ocr.py index 82297d35170..cbe444d44a6 100644 --- a/litellm/rust_bridge/ocr.py +++ b/litellm/rust_bridge/ocr.py @@ -2,16 +2,16 @@ from __future__ import annotations -import os from collections.abc import Awaitable -from typing import TYPE_CHECKING, Any, Final, Protocol, cast +from typing import Final, Protocol, cast import httpx +from litellm.rust_bridge import configuration as _configuration from litellm.rust_bridge.timeouts import timeout_to_seconds as _timeout_to_seconds -if TYPE_CHECKING: - from litellm.rust_bridge.messages import RustAmessages, RustMessages +rust_ocr_enabled = _configuration.rust_ocr_enabled +use_litellm_rust = _configuration.use_litellm_rust class RustOcr(Protocol): @@ -51,69 +51,20 @@ class _Unset: _UNSET: Final[_Unset] = _Unset() -def _env_enables_rust_ocr() -> bool: - return os.getenv("LITELLM_USE_RUST_OCR", "").strip().lower() in { - "1", - "true", - "yes", - "on", - } - - -_rust_ocr_enabled = _env_enables_rust_ocr() _rust_ocr_impl: RustOcr | None = None _rust_aocr_impl: RustAocr | None = None -def use_litellm_rust( - enabled: bool = True, +def set_rust_ocr( *, ocr: RustOcr | None | _Unset = _UNSET, aocr: RustAocr | None | _Unset = _UNSET, - messages: RustMessages | None | _Unset = _UNSET, - amessages: RustAmessages | None | _Unset = _UNSET, - responses_websocket: Any | None | _Unset = _UNSET, - transcription: Any | None | _Unset = _UNSET, - atranscription: Any | None | _Unset = _UNSET, ) -> None: - global _rust_ocr_enabled, _rust_ocr_impl, _rust_aocr_impl - configuring_ocr: Final = not isinstance(ocr, _Unset) or not isinstance(aocr, _Unset) - configuring_messages: Final = not isinstance(messages, _Unset) or not isinstance(amessages, _Unset) - configuring_responses_websocket: Final = not isinstance(responses_websocket, _Unset) - configuring_transcription: Final = not isinstance(transcription, _Unset) or not isinstance(atranscription, _Unset) - if configuring_ocr or (not configuring_messages and not configuring_responses_websocket): - _rust_ocr_enabled = enabled + global _rust_ocr_impl, _rust_aocr_impl if not isinstance(ocr, _Unset): _rust_ocr_impl = ocr if not isinstance(aocr, _Unset): _rust_aocr_impl = aocr - if configuring_transcription: - from litellm.rust_bridge.transcription import configure_rust_transcription - - configure_rust_transcription( - enabled=enabled, - transcription=transcription, - atranscription=atranscription, - ) - if not configuring_messages and not configuring_responses_websocket: - return - if configuring_messages: - from litellm.rust_bridge.messages import set_rust_messages - - if not isinstance(messages, _Unset) and not isinstance(amessages, _Unset): - set_rust_messages(messages=messages, amessages=amessages) - elif not isinstance(messages, _Unset): - set_rust_messages(messages=messages) - else: - set_rust_messages(amessages=amessages) - if configuring_responses_websocket: - from litellm.rust_bridge.responses_websocket import set_rust_responses_websocket - - set_rust_responses_websocket(connection=responses_websocket) - - -def rust_ocr_enabled() -> bool: - return _rust_ocr_enabled def load_rust_ocr() -> RustOcr | None: diff --git a/litellm/rust_bridge/runtime.py b/litellm/rust_bridge/runtime.py new file mode 100644 index 00000000000..0bf60b1aa0b --- /dev/null +++ b/litellm/rust_bridge/runtime.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from enum import Enum +from typing import Final, Generic, NoReturn, TypeAlias, TypeVar, cast + +from litellm.exceptions import APIError +from litellm.rust_bridge.bindings import native_exception_types + +NativeT = TypeVar("NativeT") +ResultT = TypeVar("ResultT") + + +class FallbackMode(Enum): + PYTHON = "python" + RUST_REQUIRED = "rust_required" + + +@dataclass(frozen=True, slots=True) +class RustHandled(Generic[ResultT]): + value: ResultT + + +@dataclass(frozen=True, slots=True) +class RustDeclined: + reason: str + + +@dataclass(frozen=True, slots=True) +class RustUnavailable: + pass + + +RustAttempt: TypeAlias = RustHandled[ResultT] | RustDeclined | RustUnavailable + + +@dataclass(frozen=True, slots=True) +class BridgeErrorContext: + route: str + provider: str + model: str + + +def invoke( + *, + native_call: Callable[[], NativeT] | None, + fallback: Callable[[], ResultT], + adapt: Callable[[NativeT], ResultT], + mode: FallbackMode, + context: BridgeErrorContext, +) -> ResultT: + result: Final = attempt(native_call=native_call, adapt=adapt, context=context) + if isinstance(result, RustHandled): + return result.value + if mode is FallbackMode.PYTHON: + return fallback() + _raise_required(result, context) + + +async def ainvoke( + *, + native_call: Callable[[], Awaitable[NativeT]] | None, + fallback: Callable[[], Awaitable[ResultT]], + adapt: Callable[[NativeT], ResultT], + mode: FallbackMode, + context: BridgeErrorContext, +) -> ResultT: + result: Final = await aattempt(native_call=native_call, adapt=adapt, context=context) + if isinstance(result, RustHandled): + return result.value + if mode is FallbackMode.PYTHON: + return await fallback() + _raise_required(result, context) + + +def attempt( + *, + native_call: Callable[[], NativeT] | None, + adapt: Callable[[NativeT], ResultT], + context: BridgeErrorContext, +) -> RustAttempt[ResultT]: + if native_call is None: + return RustUnavailable() + exceptions: Final = native_exception_types() + if exceptions is None: + return RustHandled(adapt(native_call())) + declined, upstream = exceptions + try: + value: Final = native_call() + except declined as error: + return RustDeclined(reason=_decline_reason(error)) + except upstream as error: + _raise_upstream(error, context) + return RustHandled(adapt(value)) + + +async def aattempt( + *, + native_call: Callable[[], Awaitable[NativeT]] | None, + adapt: Callable[[NativeT], ResultT], + context: BridgeErrorContext, +) -> RustAttempt[ResultT]: + if native_call is None: + return RustUnavailable() + exceptions: Final = native_exception_types() + if exceptions is None: + return RustHandled(adapt(await native_call())) + declined, upstream = exceptions + try: + value: Final = await native_call() + except declined as error: + return RustDeclined(reason=_decline_reason(error)) + except upstream as error: + _raise_upstream(error, context) + return RustHandled(adapt(value)) + + +def call(operation: Callable[[], ResultT], context: BridgeErrorContext) -> ResultT: + exceptions: Final = native_exception_types() + if exceptions is None: + return operation() + upstream: Final = exceptions[1] + try: + return operation() + except upstream as error: + _raise_upstream(error, context) + + +async def acall(operation: Callable[[], Awaitable[ResultT]], context: BridgeErrorContext) -> ResultT: + exceptions: Final = native_exception_types() + if exceptions is None: + return await operation() + upstream: Final = exceptions[1] + try: + return await operation() + except upstream as error: + _raise_upstream(error, context) + + +def _decline_reason(error: BaseException) -> str: + reason: Final = error.args[0] if error.args else str(error) + return reason if isinstance(reason, str) else str(reason) + + +def _raise_required( + result: RustDeclined | RustUnavailable, + context: BridgeErrorContext, +) -> NoReturn: + raise RuntimeError(f"Rust {context.route} bridge {_required_reason(result)}") + + +def _required_reason(result: RustDeclined | RustUnavailable) -> str: + match result: + case RustUnavailable(): + return "is unavailable" + case RustDeclined(reason=reason): + return f"declined the request: {reason}" + + +def _raise_upstream(error: BaseException, context: BridgeErrorContext) -> NoReturn: + args: Final = cast(tuple[object, ...], error.args) + status_value: Final = args[0] if args else 0 + message_value: Final = args[1] if len(args) > 1 else str(error) + status: Final = status_value if isinstance(status_value, int) else 0 + message: Final = message_value if isinstance(message_value, str) else str(message_value) + raise APIError( + status_code=status or 500, + message=f"litellm rust {context.route}: {message}", + llm_provider=context.provider, + model=context.model, + ) from error + + +def identity(value: ResultT) -> ResultT: + return value + + +async def async_none() -> None: + return None diff --git a/litellm/types/router.py b/litellm/types/router.py index 2a5f264cee3..4f4df1a8d2e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -305,6 +305,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): """ custom_llm_provider: str | None = None + rust: bool | None = None tpm: int | None = None rpm: int | None = None itpm: int | None = None diff --git a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py index fbd7e36e298..293f75b7592 100644 --- a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py +++ b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py @@ -8,6 +8,7 @@ import pytest import litellm from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.rust_bridge import configuration from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -109,10 +110,12 @@ class RaisingAsyncMessages: @pytest.fixture(autouse=True) def _reset_rust_flag(): - litellm.use_litellm_rust(False, messages=None, amessages=None) + rust_messages.set_rust_messages(messages=None, amessages=None) + configuration.reset_rust_configuration() rust_bridge_loader._cached_bridge = rust_bridge_loader._BRIDGE_SENTINEL yield - litellm.use_litellm_rust(False, messages=None, amessages=None) + rust_messages.set_rust_messages(messages=None, amessages=None) + configuration.reset_rust_configuration() rust_bridge_loader._cached_bridge = rust_bridge_loader._BRIDGE_SENTINEL @@ -122,17 +125,6 @@ def test_load_rust_messages_returns_injected_impl(): assert rust_messages.load_rust_messages() is bridge -def test_configuring_messages_does_not_enable_ocr(): - from litellm.rust_bridge.ocr import rust_ocr_enabled - - litellm.use_litellm_rust(False) - assert rust_ocr_enabled() is False - - litellm.use_litellm_rust(True, messages=RecordingMessages()) - - assert rust_ocr_enabled() is False - - def test_bare_use_litellm_rust_still_toggles_ocr(): from litellm.rust_bridge.ocr import rust_ocr_enabled @@ -264,7 +256,7 @@ async def test_gate_falls_back_to_python_when_bridge_raises(): @pytest.mark.asyncio async def test_gate_skips_rust_when_flag_absent(): bridge = ExplodingAsyncMessages() - litellm.use_litellm_rust(True, amessages=bridge) + rust_messages.set_rust_messages(amessages=bridge) response = await _gate(litellm_params=GenericLiteLLMParams(api_key="sk-azure")) @@ -272,6 +264,18 @@ async def test_gate_skips_rust_when_flag_absent(): assert bridge.calls == 0 +@pytest.mark.asyncio +async def test_gate_uses_process_enable_without_request_override(): + bridge = RecordingAsyncMessages() + rust_messages.set_rust_messages(amessages=bridge) + litellm.use_litellm_rust(True) + + response = await _gate(litellm_params=GenericLiteLLMParams(api_key="sk-azure")) + + assert response is not None + assert bridge.calls[0]["custom_llm_provider"] == "azure_ai" + + @pytest.mark.asyncio async def test_gate_skips_rust_when_flag_false(): bridge = ExplodingAsyncMessages() @@ -305,7 +309,7 @@ async def test_gate_invokes_rust_for_native_anthropic_provider(): @pytest.mark.asyncio async def test_gate_invokes_rust_when_env_var_set(monkeypatch): bridge = RecordingAsyncMessages() - litellm.use_litellm_rust(True, amessages=bridge) + rust_messages.set_rust_messages(amessages=bridge) monkeypatch.setenv("LITELLM_RUST", "1") response = await _gate( @@ -320,7 +324,7 @@ async def test_gate_invokes_rust_when_env_var_set(monkeypatch): @pytest.mark.asyncio async def test_gate_env_var_falsey_does_not_enable(monkeypatch): bridge = ExplodingAsyncMessages() - litellm.use_litellm_rust(True, amessages=bridge) + rust_messages.set_rust_messages(amessages=bridge) monkeypatch.setenv("LITELLM_RUST", "0") response = await _gate( diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index acad249a2bb..0764aec7185 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -1,7 +1,7 @@ """Tests for the optional Rust-backed OCR path.""" -import importlib import builtins +import importlib import types from typing import Any @@ -10,6 +10,7 @@ import pytest import litellm from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.rust_bridge import configuration # `litellm/__init__.py` does `from .ocr.main import *`, which binds the `ocr` # function onto `litellm.ocr` and shadows the submodule, so import the modules @@ -214,10 +215,12 @@ def build_prepared_request( @pytest.fixture(autouse=True) def _reset_rust_flag(): """Keep the global toggle isolated between tests.""" - rust_bridge.use_litellm_rust(False, ocr=None, aocr=None) + rust_bridge.set_rust_ocr(ocr=None, aocr=None) + configuration.reset_rust_configuration() rust_bridge_loader._cached_bridge = rust_bridge_loader._BRIDGE_SENTINEL yield - rust_bridge.use_litellm_rust(False, ocr=None, aocr=None) + rust_bridge.set_rust_ocr(ocr=None, aocr=None) + configuration.reset_rust_configuration() rust_bridge_loader._cached_bridge = rust_bridge_loader._BRIDGE_SENTINEL @@ -247,7 +250,14 @@ def test_use_litellm_rust_toggles_flag(): def test_env_var_enables_rust_ocr(monkeypatch): monkeypatch.setenv("LITELLM_USE_RUST_OCR", "1") - assert rust_bridge._env_enables_rust_ocr() is True + with pytest.warns(DeprecationWarning, match="LITELLM_USE_RUST_OCR is deprecated"): + assert rust_bridge.rust_ocr_enabled() is True + + +def test_explicit_false_overrides_process_enable(): + litellm.use_litellm_rust(True) + + assert ocr_main._rust_ocr_enabled(build_prepared_request(litellm_params={"rust": False})) is False def test_load_rust_ocr_returns_injected_impl(): @@ -471,9 +481,7 @@ def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing(): ocr_main._run_rust_ocr( prepared_request=build_prepared_request(api_key=None, timeout=None), - resolve_api_key=lambda name: ( - "sk-from-vault" if name == "MISTRAL_API_KEY" else None - ), + resolve_api_key=lambda name: "sk-from-vault" if name == "MISTRAL_API_KEY" else None, ) assert bridge.calls[0]["api_key"] == "sk-from-vault" @@ -580,9 +588,7 @@ def test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager(): api_base=None, timeout=None, ), - resolve_api_key=lambda name: ( - "https://azure.example.com" if name == "AZURE_AI_API_BASE" else None - ), + resolve_api_key=lambda name: "https://azure.example.com" if name == "AZURE_AI_API_BASE" else None, ) assert bridge.calls[0]["api_base"] == "https://azure.example.com" @@ -600,9 +606,7 @@ def test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint(): timeout=None, ), resolve_api_key=lambda name: ( - "https://document-intelligence.example.com" - if name == "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT" - else None + "https://document-intelligence.example.com" if name == "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT" else None ), ) @@ -815,9 +819,6 @@ def test_ocr_provider_configs_expose_api_key_env_vars(): assert BaseOCRConfig().get_api_key_env_var() is None assert MistralOCRConfig().get_api_key_env_var() == "MISTRAL_API_KEY" assert AzureAIOCRConfig().get_api_key_env_var() == "AZURE_AI_API_KEY" - assert ( - AzureDocumentIntelligenceOCRConfig().get_api_key_env_var() - == "AZURE_DOCUMENT_INTELLIGENCE_API_KEY" - ) + assert AzureDocumentIntelligenceOCRConfig().get_api_key_env_var() == "AZURE_DOCUMENT_INTELLIGENCE_API_KEY" assert VertexAIOCRConfig().get_api_key_env_var() == "VERTEX_AI_API_KEY" assert VertexAIDeepSeekOCRConfig().get_api_key_env_var() == "VERTEX_AI_API_KEY" diff --git a/tests/test_litellm/responses/test_rust_bridge_websocket.py b/tests/test_litellm/responses/test_rust_bridge_websocket.py index c9a5b988be6..1233ddf1785 100644 --- a/tests/test_litellm/responses/test_rust_bridge_websocket.py +++ b/tests/test_litellm/responses/test_rust_bridge_websocket.py @@ -3,7 +3,7 @@ from __future__ import annotations import pytest from litellm.llms.custom_httpx.llm_http_handler import _rust_responses_websocket_enabled -from litellm.rust_bridge import responses_websocket +from litellm.rust_bridge import configuration, responses_websocket from litellm.types.router import GenericLiteLLMParams @@ -39,12 +39,33 @@ class _FakeNativeBridge: return _FakeNativeConnection() +@pytest.fixture(autouse=True) +def reset_responses_websocket(): + responses_websocket.set_rust_responses_websocket(connection=None) + configuration.reset_rust_configuration() + yield + responses_websocket.set_rust_responses_websocket(connection=None) + configuration.reset_rust_configuration() + + def test_rust_websocket_bridge_is_disabled_without_flag() -> None: assert not _rust_responses_websocket_enabled("openai", GenericLiteLLMParams()) assert not _rust_responses_websocket_enabled("anthropic", GenericLiteLLMParams(rust=True)) assert _rust_responses_websocket_enabled("openai", GenericLiteLLMParams(rust=True)) +def test_explicit_false_overrides_process_enable() -> None: + configuration.use_litellm_rust(True) + + assert not _rust_responses_websocket_enabled("openai", GenericLiteLLMParams(rust=False)) + + +def test_process_enable_applies_without_request_override() -> None: + configuration.use_litellm_rust(True) + + assert _rust_responses_websocket_enabled("openai", GenericLiteLLMParams()) + + @pytest.mark.asyncio async def test_adapter_raises_clean_close_when_rust_connection_ends() -> None: adapter = responses_websocket._ConnectionAdapter(_ClosedNativeConnection()) diff --git a/tests/test_litellm/rust_bridge/test_bindings.py b/tests/test_litellm/rust_bridge/test_bindings.py new file mode 100644 index 00000000000..6ce48b3bdf0 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_bindings.py @@ -0,0 +1,21 @@ +from types import SimpleNamespace + +from litellm.rust_bridge import bindings + + +def test_binding_distinguishes_disable_from_reset(monkeypatch) -> None: + native = SimpleNamespace(route=lambda: "native") + monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) + binding: bindings.NativeBinding[object] = bindings.NativeBinding("route") + + assert binding.load() is native.route + + binding.override(None) + assert binding.load() is None + + replacement = object() + binding.override(replacement) + assert binding.load() is replacement + + binding.reset() + assert binding.load() is native.route diff --git a/tests/test_litellm/rust_bridge/test_chat_completions.py b/tests/test_litellm/rust_bridge/test_chat_completions.py index 47cb66932b7..03921133c77 100644 --- a/tests/test_litellm/rust_bridge/test_chat_completions.py +++ b/tests/test_litellm/rust_bridge/test_chat_completions.py @@ -11,6 +11,7 @@ import pytest import litellm from litellm.rust_bridge import chat_completions as bridge +from litellm.rust_bridge import configuration from litellm.types.utils import ModelResponse RUST_RESPONSE = { @@ -68,13 +69,11 @@ def _hide_native_bridge(monkeypatch): @pytest.fixture(autouse=True) def reset_bridge(): """Every test starts with no injected callables, and leaves none behind.""" - bridge.set_rust_chat_completions( - chat_completions=None, achat_completions=None, decline=None - ) + bridge.set_rust_chat_completions(chat_completions=None, achat_completions=None, decline=None) + configuration.reset_rust_configuration() yield - bridge.set_rust_chat_completions( - chat_completions=None, achat_completions=None, decline=None - ) + bridge.set_rust_chat_completions(chat_completions=None, achat_completions=None, decline=None) + configuration.reset_rust_configuration() class _RecordingDecline: @@ -138,6 +137,18 @@ class TestGate: assert gate.calls[0]["model"] == "claude-sonnet-4-5" assert gate.calls[0]["custom_llm_provider"] == "anthropic" + def test_explicit_false_overrides_process_enable(self): + bridge.set_rust_chat_completions(decline=_RecordingDecline()) + configuration.use_litellm_rust(True) + + assert _accepts(litellm_params={"rust": False}) is False + + def test_process_enable_applies_without_request_override(self): + bridge.set_rust_chat_completions(decline=_RecordingDecline()) + configuration.use_litellm_rust(True) + + assert _accepts(litellm_params={}) is True + def test_the_env_var_opts_in_without_a_per_model_flag(self, monkeypatch): monkeypatch.setenv("LITELLM_RUST", "true") bridge.set_rust_chat_completions(decline=_RecordingDecline()) @@ -253,9 +264,7 @@ class TestSyncCall: assert result.usage.completion_tokens == 4 assert result.usage.total_tokens == 15 assert result._hidden_params["additional_headers"] == {"x-litellm-rust": "true"} - assert result.id == original_id, ( - "the rust path must keep the chatcmpl id litellm already minted" - ) + assert result.id == original_id, "the rust path must keep the chatcmpl id litellm already minted" def test_passes_the_timeout_through_as_seconds(self): native = _RecordingCall() @@ -269,9 +278,7 @@ class TestSyncCall: def test_falls_back_when_the_core_declines_before_calling_the_provider(self, monkeypatch): _fake_native_bridge(monkeypatch) - bridge.set_rust_chat_completions( - chat_completions=_RecordingCall(error=_FakeDeclined("streaming")) - ) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeDeclined("streaming"))) assert bridge.chat_completions(**_call_kwargs(ModelResponse())) is None @@ -290,13 +297,9 @@ class TestAsyncCall: assert await bridge.achat_completions(**_call_kwargs(ModelResponse())) is None @pytest.mark.asyncio - async def test_falls_back_when_the_core_declines_before_calling_the_provider( - self, monkeypatch - ): + async def test_falls_back_when_the_core_declines_before_calling_the_provider(self, monkeypatch): _fake_native_bridge(monkeypatch) - bridge.set_rust_chat_completions( - achat_completions=_RecordingAsyncCall(error=_FakeDeclined("streaming")) - ) + bridge.set_rust_chat_completions(achat_completions=_RecordingAsyncCall(error=_FakeDeclined("streaming"))) assert await bridge.achat_completions(**_call_kwargs(ModelResponse())) is None @@ -310,25 +313,19 @@ class TestAsyncFallbackWrapper: ran.append(True) return "python" - result = await bridge.achat_completions_or_fallback( - **_call_kwargs(ModelResponse()), python_fallback=fallback - ) + result = await bridge.achat_completions_or_fallback(**_call_kwargs(ModelResponse()), python_fallback=fallback) assert result.choices[0].message.content == "hello from rust" assert ran == [] @pytest.mark.asyncio async def test_runs_the_fallback_when_the_core_declines(self, monkeypatch): _fake_native_bridge(monkeypatch) - bridge.set_rust_chat_completions( - achat_completions=_RecordingAsyncCall(error=_FakeDeclined("streaming")) - ) + bridge.set_rust_chat_completions(achat_completions=_RecordingAsyncCall(error=_FakeDeclined("streaming"))) async def fallback(): return "python" - result = await bridge.achat_completions_or_fallback( - **_call_kwargs(ModelResponse()), python_fallback=fallback - ) + result = await bridge.achat_completions_or_fallback(**_call_kwargs(ModelResponse()), python_fallback=fallback) assert result == "python" @pytest.mark.asyncio @@ -338,9 +335,7 @@ class TestAsyncFallbackWrapper: async def fallback(): return "python" - result = await bridge.achat_completions_or_fallback( - **_call_kwargs(ModelResponse()), python_fallback=fallback - ) + result = await bridge.achat_completions_or_fallback(**_call_kwargs(ModelResponse()), python_fallback=fallback) assert result == "python" @@ -353,17 +348,13 @@ class TestFailureClassification: _fake_native_bridge(monkeypatch) def test_a_decline_falls_back_because_nothing_was_sent(self): - bridge.set_rust_chat_completions( - chat_completions=_RecordingCall(error=_FakeDeclined("streaming")) - ) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeDeclined("streaming"))) assert bridge.chat_completions(**_call_kwargs(ModelResponse())) is None def test_an_upstream_failure_is_surfaced_with_its_status(self): from litellm.exceptions import APIError - bridge.set_rust_chat_completions( - chat_completions=_RecordingCall(error=_FakeUpstream(429, "429: rate limited")) - ) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeUpstream(429, "429: rate limited"))) with pytest.raises(APIError) as raised: bridge.chat_completions(**_call_kwargs(ModelResponse())) assert raised.value.status_code == 429 @@ -372,17 +363,13 @@ class TestFailureClassification: def test_a_transport_failure_with_no_response_surfaces_as_a_500(self): from litellm.exceptions import APIError - bridge.set_rust_chat_completions( - chat_completions=_RecordingCall(error=_FakeUpstream(0, "connection reset")) - ) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeUpstream(0, "connection reset"))) with pytest.raises(APIError) as raised: bridge.chat_completions(**_call_kwargs(ModelResponse())) assert raised.value.status_code == 500 def test_an_unrecognized_error_is_not_swallowed(self): - bridge.set_rust_chat_completions( - chat_completions=_RecordingCall(error=RuntimeError("something else")) - ) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=RuntimeError("something else"))) with pytest.raises(RuntimeError): bridge.chat_completions(**_call_kwargs(ModelResponse())) @@ -390,9 +377,7 @@ class TestFailureClassification: async def test_the_async_wrapper_does_not_fall_back_on_an_upstream_failure(self): from litellm.exceptions import APIError - bridge.set_rust_chat_completions( - achat_completions=_RecordingAsyncCall(error=_FakeUpstream(500, "500: boom")) - ) + bridge.set_rust_chat_completions(achat_completions=_RecordingAsyncCall(error=_FakeUpstream(500, "500: boom"))) ran = [] async def fallback(): @@ -400,9 +385,7 @@ class TestFailureClassification: return "python" with pytest.raises(APIError): - await bridge.achat_completions_or_fallback( - **_call_kwargs(ModelResponse()), python_fallback=fallback - ) + await bridge.achat_completions_or_fallback(**_call_kwargs(ModelResponse()), python_fallback=fallback) assert ran == [], "a request the provider already served must not be re-issued" @pytest.mark.asyncio @@ -414,7 +397,5 @@ class TestFailureClassification: async def fallback(): return "python" - result = await bridge.achat_completions_or_fallback( - **_call_kwargs(ModelResponse()), python_fallback=fallback - ) + result = await bridge.achat_completions_or_fallback(**_call_kwargs(ModelResponse()), python_fallback=fallback) assert result == "python" diff --git a/tests/test_litellm/rust_bridge/test_configuration.py b/tests/test_litellm/rust_bridge/test_configuration.py new file mode 100644 index 00000000000..e7ce9a625ad --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_configuration.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from collections.abc import Generator +from typing import Final + +import pytest + +from litellm.rust_bridge import configuration +from litellm.rust_bridge import ocr as rust_ocr + + +class _OcrBridge: + def __call__( + self, + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str | None, + extra_headers: dict[str, object] | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> dict[str, object]: + return {} + + +@pytest.fixture(autouse=True) +def _isolated_configuration( # pyright: ignore[reportUnusedFunction] # pytest discovers fixtures dynamically + monkeypatch: pytest.MonkeyPatch, +) -> Generator[None]: + configuration.reset_rust_configuration() + monkeypatch.delenv("LITELLM_RUST", raising=False) + monkeypatch.delenv("LITELLM_USE_RUST_OCR", raising=False) + rust_ocr.set_rust_ocr(ocr=None, aocr=None) + yield + configuration.reset_rust_configuration() + rust_ocr.set_rust_ocr(ocr=None, aocr=None) + + +@pytest.mark.parametrize( + ("request_override", "process", "environment", "legacy_ocr", "release_default", "expected"), + ( + (False, True, True, True, True, False), + (True, False, False, False, False, True), + (None, False, True, True, True, False), + (None, True, False, False, False, True), + (None, None, False, True, True, False), + (None, None, True, False, False, True), + (None, None, None, False, True, False), + (None, None, None, True, False, True), + (None, None, None, None, False, False), + (None, None, None, None, True, True), + ), +) +def test_resolution_precedence( + request_override: bool | None, + process: bool | None, + environment: bool | None, + legacy_ocr: bool | None, + release_default: bool, + expected: bool, +) -> None: + assert ( + configuration.resolve_rust_enabled( + request_override=request_override, + process_override=process, + environment_override=environment, + legacy_ocr_override=legacy_ocr, + release_default=release_default, + ) + is expected + ) + + +def test_release_default_remains_disabled() -> None: + assert configuration.DEFAULT_RUST_ENABLED is False + assert configuration.rust_enabled() is False + + +def test_process_override_wins_over_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "0") + configuration.use_litellm_rust(True) + + assert configuration.rust_enabled() is True + assert configuration.rust_enabled(request_override=False) is False + + +def test_global_environment_accepts_explicit_false(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "off") + + assert configuration.rust_enabled() is False + + +def test_invalid_environment_value_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "sometimes") + + with pytest.raises(ValueError, match="LITELLM_RUST must be one of"): + configuration.rust_enabled() + + +def test_explicit_override_precedes_invalid_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "sometimes") + + assert configuration.rust_enabled(request_override=False) is False + configuration.use_litellm_rust(True) + assert configuration.rust_enabled() is True + + +def test_legacy_ocr_environment_is_deprecated_and_ocr_only(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_USE_RUST_OCR", "1") + + with pytest.warns(DeprecationWarning, match="LITELLM_USE_RUST_OCR is deprecated"): + assert configuration.rust_ocr_enabled() is True + assert configuration.rust_enabled() is False + + +def test_global_environment_precedes_legacy_ocr_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "0") + monkeypatch.setenv("LITELLM_USE_RUST_OCR", "1") + + assert configuration.rust_ocr_enabled() is False + + +def test_deprecated_public_injection_delegates_to_internal_binding() -> None: + bridge: Final = _OcrBridge() + + with pytest.warns(DeprecationWarning, match="Injecting Rust bridge implementations"): + configuration.use_litellm_rust(True, ocr=bridge) + + assert rust_ocr.load_rust_ocr() is bridge + + +@pytest.mark.parametrize(("value", "expected"), (("1", "True"), ("0", "False"))) +def test_environment_controls_startup(value: str, expected: str) -> None: + environment: Final = {**os.environ, "LITELLM_RUST": value} + result: Final = subprocess.run( + ( + sys.executable, + "-c", + "from litellm.rust_bridge.configuration import rust_enabled; print(rust_enabled())", + ), + check=True, + capture_output=True, + text=True, + env=environment, + ) + + assert result.stdout.strip() == expected diff --git a/tests/test_litellm/rust_bridge/test_runtime.py b/tests/test_litellm/rust_bridge/test_runtime.py new file mode 100644 index 00000000000..b0fa510069b --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_runtime.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from litellm.exceptions import APIError +from litellm.rust_bridge import bindings, runtime + + +class RustBridgeDeclined(Exception): + pass + + +class RustUpstreamError(Exception): + pass + + +@pytest.fixture(autouse=True) +def native_exceptions(monkeypatch: pytest.MonkeyPatch) -> None: + native = SimpleNamespace( + RustBridgeDeclined=RustBridgeDeclined, + RustUpstreamError=RustUpstreamError, + ) + monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) + + +def context() -> runtime.BridgeErrorContext: + return runtime.BridgeErrorContext(route="messages", provider="anthropic", model="model") + + +def test_invoke_tags_native_decline_before_running_fallback() -> None: + calls: list[str] = [] + + def decline() -> object: + calls.append("rust") + raise RustBridgeDeclined("unsupported") + + value = runtime.invoke( + native_call=decline, + fallback=lambda: calls.append("python") or "fallback", + adapt=str, + mode=runtime.FallbackMode.PYTHON, + context=context(), + ) + + assert value == "fallback" + assert calls == ["rust", "python"] + + +def test_invoke_translates_upstream_without_fallback() -> None: + def fail() -> object: + raise RustUpstreamError(429, "rate limited") + + with pytest.raises(APIError, match="rate limited") as caught: + runtime.invoke( + native_call=fail, + fallback=lambda: pytest.fail("fallback must not run"), + adapt=str, + mode=runtime.FallbackMode.PYTHON, + context=context(), + ) + + assert caught.value.status_code == 429 + + +@pytest.mark.asyncio +async def test_ainvoke_handles_native_success() -> None: + async def native() -> int: + return 3 + + async def fallback() -> str: + pytest.fail("fallback must not run") + + assert ( + await runtime.ainvoke( + native_call=native, + fallback=fallback, + adapt=str, + mode=runtime.FallbackMode.PYTHON, + context=context(), + ) + == "3" + ) + + +def test_required_mode_rejects_unavailable_bridge() -> None: + with pytest.raises(RuntimeError, match="is unavailable"): + runtime.invoke( + native_call=None, + fallback=lambda: pytest.fail("fallback must not run"), + adapt=str, + mode=runtime.FallbackMode.RUST_REQUIRED, + context=context(), + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bde7fd611d5..e92376ae450 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29236,6 +29236,8 @@ export interface components { regional_processing_uplift_multiplier_us?: number | null; /** Rpm */ rpm?: number | null; + /** Rust */ + rust?: boolean | null; /** S3 Bucket Name */ s3_bucket_name?: string | null; /** S3 Encryption Key Id */ @@ -39067,6 +39069,8 @@ export interface components { regional_processing_uplift_multiplier_us?: number | null; /** Rpm */ rpm?: number | null; + /** Rust */ + rust?: boolean | null; /** S3 Bucket Name */ s3_bucket_name?: string | null; /** S3 Encryption Key Id */ From 9ec04cc7b53e5dcc391f186f3fe0a21dfecf17b8 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 20:55:21 -0700 Subject: [PATCH 58/67] fix(python-bridge): prevent fallback after native execution starts --- .../src/audio_transcription/hooks.rs | 1 - .../crates/ai-gateway/src/ocr/hooks.rs | 1 - .../ai-gateway/src/routes/messages/mod.rs | 1 - .../core/src/chat_completions/handler.rs | 31 +--- .../crates/core/src/chat_completions/tests.rs | 36 +--- litellm-rust/crates/core/src/error.rs | 46 ++++- litellm-rust/crates/core/src/http_utils.rs | 4 + .../crates/core/src/messages/handler.rs | 165 +++++++++++++++++- .../crates/core/src/messages/prepare.rs | 5 +- .../crates/core/src/messages/tests.rs | 5 +- .../crates/python-bridge/src/errors.rs | 73 ++++++-- .../src/routes/chat_completions.rs | 4 +- .../python-bridge/src/routes/definition.rs | 29 +++ .../python-bridge/src/routes/messages.rs | 4 +- litellm/llms/custom_httpx/llm_http_handler.py | 29 ++- litellm/rust_bridge/messages.py | 115 ++++++------ .../test_rust_bridge_messages.py | 73 +++++++- .../rust_bridge/native_route_wheel_test.py | 133 +++++++++++++- 18 files changed, 574 insertions(+), 181 deletions(-) diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs index dbe2d3a325b..42ffd4cbcff 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs @@ -277,7 +277,6 @@ fn core_error_kind(error: &Error) -> &'static str { Error::Http { .. } => "HttpError", Error::InvalidResponse(_) => "InvalidResponse", Error::Network(_) => "NetworkError", - Error::Connect(_) => "ConnectError", Error::Routing(_) => "RoutingError", Error::Unsupported(_) => "UnsupportedRequest", } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs index 401e26d3b29..6343122b9e9 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs @@ -321,7 +321,6 @@ fn core_error_kind(error: &Error) -> &'static str { Error::Http { .. } => "HttpError", Error::InvalidResponse(_) => "InvalidResponse", Error::Network(_) => "NetworkError", - Error::Connect(_) => "ConnectError", Error::Routing(_) => "RoutingError", Error::Unsupported(_) => "UnsupportedRequest", } diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs index e9f8c477f36..e4861c3954c 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs @@ -105,7 +105,6 @@ impl IntoResponse for MessagesRouteError { ), Error::Http { .. } | Error::Network(_) - | Error::Connect(_) | Error::InvalidResponse(_) | Error::InvalidType { .. } | Error::MissingField(_) => ( diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index 7e2731442cc..f282e82c4fa 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,7 +1,7 @@ use serde_json::Value; -use crate::error::Error; -use crate::http_utils::truncate_error_body; +use crate::error::{Error, as_response_error}; +use crate::http_utils::{classify_send_error, truncate_error_body}; use super::client::http_client; use super::transformation::ChatCompletionsAuth; @@ -27,16 +27,7 @@ pub(super) async fn execute_chat_completions_provider_call( request_builder = request_builder.timeout(duration); } - let response = request_builder.send().await.map_err(|err| { - // Failing to establish the connection means the request never went out, - // so the host can still serve it. Everything else here, a timeout - // above all, may have reached the provider and been answered. - if err.is_connect() || err.is_builder() { - Error::Connect(err.to_string()) - } else { - Error::Network(err.to_string()) - } - })?; + let response = request_builder.send().await.map_err(classify_send_error)?; let status = response.status(); let text = response @@ -60,22 +51,6 @@ pub(super) async fn execute_chat_completions_provider_call( .map_err(as_response_error) } -/// Re-tag an error raised while normalizing a response the provider already -/// returned. -/// -/// A config reports the same variants on either side of the call: a missing -/// field or an unsupported block can mean "this request cannot be translated" -/// during prepare and "this response cannot be normalized" here. Only the -/// second kind has already been billed, and a host that keeps a reference -/// implementation must not retry those, so collapse them to one variant that -/// can only mean the provider was already called. -pub(super) fn as_response_error(err: Error) -> Error { - match err { - already @ (Error::InvalidResponse(_) | Error::Http { .. }) => already, - other => Error::InvalidResponse(other.to_string()), - } -} - #[cfg(feature = "bedrock-auth")] pub(super) async fn signed_headers( request: &ProviderChatCompletionsRequest, diff --git a/litellm-rust/crates/core/src/chat_completions/tests.rs b/litellm-rust/crates/core/src/chat_completions/tests.rs index 2858d180e27..bdd413d7cff 100644 --- a/litellm-rust/crates/core/src/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/chat_completions/tests.rs @@ -769,11 +769,7 @@ mod round_trip { } #[tokio::test] - async fn a_connection_that_is_never_established_declines_instead_of_failing() { - // Nothing was sent, so nothing was billed and the host can still serve - // the request. Classing this with the post-send failures would turn a - // recoverable fallback into a user-facing error on exactly the - // deployments whose transport is configured only on the Python client. + async fn a_connection_that_is_never_established_is_still_terminal() { let port = { let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); listener.local_addr().expect("has an address").port() @@ -787,34 +783,8 @@ mod round_trip { .await .expect_err("nothing is listening"); assert!( - matches!(err, Error::Connect(_)), - "expected a pre-send connect failure, got {err:?}" + matches!(err, Error::Network(_)), + "expected a terminal network failure, got {err:?}" ); } - - #[test] - fn response_errors_collapse_to_one_variant_that_can_only_mean_already_sent() { - use crate::chat_completions::handler::as_response_error; - - for original in [ - Error::MissingField("usage"), - Error::Unsupported("non-text response content block"), - Error::InvalidRequest("whatever".to_string()), - Error::Auth("whatever".to_string()), - ] { - let label = format!("{original:?}"); - assert!( - matches!(as_response_error(original), Error::InvalidResponse(_)), - "{label} must not stay retryable once the provider has answered" - ); - } - // An upstream status is already unambiguous, so it survives intact. - assert!(matches!( - as_response_error(Error::Http { - status: 500, - body: "boom".to_string() - }), - Error::Http { status: 500, .. } - )); - } } diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index db3fa2ec704..a5de4b722d2 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -21,13 +21,6 @@ pub enum Error { Http { status: u16, body: String }, #[error("upstream network error: {0}")] Network(String), - /// The provider was never reached: DNS, TCP, TLS or proxy setup failed - /// before any byte of the request went out. Nothing was billed, so a host - /// that keeps a reference implementation can serve the request itself. - /// A timeout is deliberately not this, since the provider may have received - /// and answered the request already. - #[error("could not reach the provider: {0}")] - Connect(String), #[error("routing error: {0}")] Routing(String), /// The request is outside the surface this route covers in Rust. Hosts that @@ -36,6 +29,14 @@ pub enum Error { Unsupported(&'static str), } +/// Re-tag an error raised after the provider has already returned a response. +pub(crate) fn as_response_error(err: Error) -> Error { + match err { + already @ (Error::InvalidResponse(_) | Error::Http { .. }) => already, + other => Error::InvalidResponse(other.to_string()), + } +} + pub fn json_type_name(value: &serde_json::Value) -> &'static str { match value { serde_json::Value::Null => "null", @@ -46,3 +47,34 @@ pub fn json_type_name(value: &serde_json::Value) -> &'static str { serde_json::Value::Object(_) => "object", } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn response_errors_collapse_to_one_non_retryable_variant() { + for original in [ + Error::MissingField("usage"), + Error::Unsupported("non-text response content block"), + Error::InvalidRequest("whatever".to_string()), + Error::Auth("whatever".to_string()), + ] { + assert!(matches!( + as_response_error(original), + Error::InvalidResponse(_) + )); + } + } + + #[test] + fn response_errors_preserve_an_upstream_status() { + assert!(matches!( + as_response_error(Error::Http { + status: 500, + body: "boom".to_string() + }), + Error::Http { status: 500, .. } + )); + } +} diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index 10661fadf96..759c7ba5652 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -5,6 +5,10 @@ use serde_json::{Map, Value}; use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS; use crate::error::{Error, json_type_name}; +pub(crate) fn classify_send_error(error: reqwest::Error) -> Error { + Error::Network(error.to_string()) +} + /// Bound an upstream error body before it crosses a host boundary, so provider /// bodies stay data-minimized. pub fn truncate_error_body(body: &str) -> String { diff --git a/litellm-rust/crates/core/src/messages/handler.rs b/litellm-rust/crates/core/src/messages/handler.rs index 13a65d86131..441f133b182 100644 --- a/litellm-rust/crates/core/src/messages/handler.rs +++ b/litellm-rust/crates/core/src/messages/handler.rs @@ -1,5 +1,6 @@ use crate::constants::ANTHROPIC_MESSAGES_PROVIDER; -use crate::error::Error; +use crate::error::{Error, as_response_error}; +use crate::http_utils::classify_send_error; use super::client::http_client; use super::common_utils::truncate_error_body; @@ -16,10 +17,7 @@ pub(super) async fn execute_messages_provider_call( request_builder = request_builder.timeout(duration); } - let response = request_builder - .send() - .await - .map_err(|err| Error::Network(err.to_string()))?; + let response = request_builder.send().await.map_err(classify_send_error)?; let status = response.status(); let text = response @@ -36,7 +34,10 @@ pub(super) async fn execute_messages_provider_call( let response = serde_json::from_str(&text) .map_err(|err| Error::InvalidResponse(format!("invalid messages response JSON: {err}")))?; - request.config.transform_response(&request.model, response) + request + .config + .transform_response(&request.model, response) + .map_err(as_response_error) } pub(super) async fn execute_messages_provider_stream( @@ -73,3 +74,155 @@ pub(super) async fn execute_messages_provider_stream( } Ok(response) } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use serde_json::json; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::{TcpListener, TcpStream}; + + use super::*; + use crate::messages::transformation::AnthropicMessagesProviderConfig; + + struct RejectingResponseConfig; + + impl AnthropicMessagesProviderConfig for RejectingResponseConfig { + fn complete_url( + &self, + _api_base: Option<&str>, + _model: &str, + _env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + unreachable!() + } + + fn resolve_api_key( + &self, + _api_key: Option<&str>, + _env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + unreachable!() + } + + fn transform_response( + &self, + _model: &str, + _response: AnthropicMessagesResponse, + ) -> Result { + Err(Error::MissingField("normalized_content")) + } + } + + static REJECTING_RESPONSE_CONFIG: RejectingResponseConfig = RejectingResponseConfig; + + fn request(url: String, timeout: Duration) -> ProviderMessagesRequest { + ProviderMessagesRequest { + provider: "anthropic".to_string(), + model: "claude-test".to_string(), + config: &REJECTING_RESPONSE_CONFIG, + url, + body: json!({}), + upstream_headers: Vec::new(), + timeout: Some(timeout), + } + } + + async fn read_http_request(socket: &mut TcpStream) -> String { + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let read = socket.read(&mut buffer).await.expect("reads request"); + if read == 0 { + break; + } + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + String::from_utf8(request).expect("request is utf8") + } + + #[tokio::test] + async fn post_response_transform_errors_are_non_retryable() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); + let addr = listener.local_addr().expect("addr"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let _ = read_http_request(&mut socket).await; + let body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-test"}"#; + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + socket + .write_all(response.as_bytes()) + .await + .expect("writes response"); + }); + + let error = execute_messages_provider_call(request( + format!("http://{addr}/v1/messages"), + Duration::from_secs(5), + )) + .await + .expect_err("response transform should fail"); + + server.await.expect("server task completes"); + assert!( + matches!(error, Error::InvalidResponse(message) if message.contains("normalized_content")) + ); + } + + #[tokio::test] + async fn refused_connections_are_terminal_network_errors() { + let port = { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); + listener.local_addr().expect("has an address").port() + }; + let error = execute_messages_provider_call(request( + format!("http://127.0.0.1:{port}"), + Duration::from_secs(1), + )) + .await + .expect_err("nothing is listening"); + + assert!(matches!(error, Error::Network(_))); + } + + #[tokio::test] + async fn established_request_timeouts_are_network_errors() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); + let addr = listener.local_addr().expect("has an address"); + let (request_received_tx, request_received_rx) = tokio::sync::oneshot::channel(); + let (release_server_tx, release_server_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let received = read_http_request(&mut socket).await; + request_received_tx.send(received).expect("reports request"); + release_server_rx.await.expect("server is released"); + }); + + let error = tokio::time::timeout( + Duration::from_secs(2), + execute_messages_provider_call(request( + format!("http://{addr}"), + Duration::from_millis(100), + )), + ) + .await + .expect("client call completes") + .expect_err("established request times out"); + + let received = tokio::time::timeout(Duration::from_secs(2), request_received_rx) + .await + .expect("server observes request") + .expect("server reports request"); + assert!(received.starts_with("POST / "), "{received}"); + release_server_tx.send(()).expect("releases server"); + server.await.expect("server task completes"); + assert!(matches!(error, Error::Network(_))); + } +} diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index 3b253ac3766..3d10a9d824f 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -25,8 +25,9 @@ pub(super) fn prepare_messages_call( let model = provider_info.model.to_string(); let provider = provider_info.custom_llm_provider; - let config = messages_provider_config(provider) - .ok_or_else(|| Error::InvalidProvider(provider.to_string()))?; + let config = messages_provider_config(provider).ok_or(Error::Unsupported( + "messages provider is not registered in the Rust bridge", + ))?; let env_lookup = |key: &str| std::env::var(key).ok(); let mut headers = string_headers(request.extra_headers)?; diff --git a/litellm-rust/crates/core/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs index df9f7051011..27dc2281d64 100644 --- a/litellm-rust/crates/core/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -437,5 +437,8 @@ async fn messages_rejects_unsupported_provider() { .await .expect_err("unsupported provider errors"); - assert!(matches!(err, Error::InvalidProvider(provider) if provider == "openai")); + assert!(matches!( + err, + Error::Unsupported("messages provider is not registered in the Rust bridge") + )); } diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 914e2e1e033..d3e47937666 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -29,28 +29,15 @@ pub(crate) fn core_error_to_pyerr(err: Error) -> PyErr { /// Map a core error for a route whose host keeps a Python implementation. /// -/// The distinction the host needs is whether the provider was already called. -/// Everything raised before the request goes out is safe for the host to retry -/// on its own path; anything after it is not, because the provider has already -/// done the work and billed for it. -pub(crate) fn chat_completions_error_to_pyerr(err: Error) -> PyErr { +/// Only an explicit capability decline permits the host to try Python. Every +/// other error may have happened after provider dispatch and must be terminal. +pub(crate) fn fallback_route_error_to_pyerr(err: Error) -> PyErr { match err { - Error::Unsupported(_) - | Error::Auth(_) - | Error::InvalidProvider(_) - | Error::InvalidRequest(_) - | Error::InvalidType { .. } - | Error::MissingField(_) - | Error::Routing(_) - // Nothing reached the provider, so serving it on Python cannot double - // bill and is the only way the caller gets an answer at all. - | Error::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), + Error::Unsupported(_) => RustBridgeDeclined::new_err(err.to_string()), Error::Http { status, body } => { RustUpstreamError::new_err((status, format!("{status}: {body}"))) } - Error::Network(message) | Error::InvalidResponse(message) => { - RustUpstreamError::new_err((0u16, message)) - } + other => RustUpstreamError::new_err((0u16, other.to_string())), } } @@ -59,3 +46,53 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add("RustBridgeDeclined", py.get_type::())?; module.add("RustUpstreamError", py.get_type::()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fallback_routes_distinguish_declines_from_upstream_failures() { + Python::initialize(); + Python::attach(|py| { + let declines = [Error::Unsupported("unsupported")]; + for error in declines { + let mapped = fallback_route_error_to_pyerr(error); + assert!(mapped.is_instance_of::(py)); + } + + let upstream_failures = [ + ( + Error::Http { + status: 429, + body: "rate limited".to_string(), + }, + (429, "429: rate limited"), + ), + ( + Error::Network("request timed out".to_string()), + (0, "upstream network error: request timed out"), + ), + ( + Error::InvalidResponse("bad JSON".to_string()), + (0, "invalid response: bad JSON"), + ), + (Error::Auth("missing key".to_string()), (0, "missing key")), + ( + Error::InvalidRequest("invalid".to_string()), + (0, "invalid request: invalid"), + ), + ]; + for (error, expected) in upstream_failures { + let mapped = fallback_route_error_to_pyerr(error); + assert!(mapped.is_instance_of::(py)); + let args: (u16, String) = mapped + .value(py) + .getattr("args") + .and_then(|args| args.extract()) + .expect("upstream error should carry status and message"); + assert_eq!(args, (expected.0, expected.1.to_string())); + } + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs index 68b7762cb10..a0257a4ade2 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs @@ -8,7 +8,7 @@ use litellm_core::chat_completions::{ use pyo3::prelude::*; use serde_json::Value; -use crate::errors::chat_completions_error_to_pyerr; +use crate::errors::fallback_route_error_to_pyerr; use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty, required_value}; fn prepare_chat_completions( @@ -86,6 +86,6 @@ bridge_route! { timeout_seconds: Option, }, prepare = prepare_chat_completions, - errors = chat_completions_error_to_pyerr, + errors = fallback_route_error_to_pyerr, extra = [chat_completions_decline], } diff --git a/litellm-rust/crates/python-bridge/src/routes/definition.rs b/litellm-rust/crates/python-bridge/src/routes/definition.rs index 5ba1cc6f1ad..d2674a80a10 100644 --- a/litellm-rust/crates/python-bridge/src/routes/definition.rs +++ b/litellm-rust/crates/python-bridge/src/routes/definition.rs @@ -401,6 +401,35 @@ asyncio.run(exercise()) }); } + #[test] + fn messages_routes_only_decline_unsupported_requests() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "routes").expect("module should be created"); + crate::routes::register(&module).expect("routes should register"); + let body = PyDict::new(py); + let kwargs = PyDict::new(py); + kwargs + .set_item("custom_llm_provider", "openai") + .expect("kwargs should accept provider"); + + let error = module + .getattr("messages") + .and_then(|function| function.call(("model", &body), Some(&kwargs))) + .expect_err("unsupported provider should decline"); + assert!(error.is_instance_of::(py)); + + kwargs + .set_item("custom_llm_provider", "anthropic") + .expect("kwargs should accept provider"); + let error = module + .getattr("messages") + .and_then(|function| function.call(("model", &body), Some(&kwargs))) + .expect_err("missing credentials should fail"); + assert!(error.is_instance_of::(py)); + }); + } + #[test] fn route_registration_rejects_duplicate_python_names() { Python::initialize(); diff --git a/litellm-rust/crates/python-bridge/src/routes/messages.rs b/litellm-rust/crates/python-bridge/src/routes/messages.rs index 2bb64a7a763..da028486e2f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages.rs @@ -5,7 +5,7 @@ use pyo3::prelude::*; use serde_json::Value; use std::future::Future; -use crate::errors::core_error_to_pyerr; +use crate::errors::fallback_route_error_to_pyerr; use crate::marshal::{RouteOptions, RouteOptionsInputs, required_value}; fn prepare_messages( @@ -61,5 +61,5 @@ bridge_route! { timeout_seconds: Option, }, prepare = prepare_messages, - errors = core_error_to_pyerr, + errors = fallback_route_error_to_pyerr, } diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 3cd6ee54069..9aaf92f62e4 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -19,6 +19,7 @@ import litellm.types.utils from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.exceptions import APIError from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, @@ -2394,27 +2395,19 @@ class BaseLLMHTTPHandler: from litellm.rust_bridge import messages as rust_messages_bridge upstream_body: Final = {key: value for key, value in request_body.items() if key != "stream"} - try: - rust_response: Final = await rust_messages_bridge.amessages( - model=model, - body=upstream_body, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=headers, - timeout=timeout, - ) - except Exception as rust_error: # noqa: BLE001 # rollout-safety fallback: any Rust bridge failure must fall back to the Python path - verbose_logger.debug( - "Rust Anthropic messages bridge raised %s; falling back to Python path", - type(rust_error).__name__, - ) - return None + rust_response: Final = await rust_messages_bridge.amessages( + model=model, + body=upstream_body, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=headers, + timeout=timeout, + ) if rust_response is None: return None response_obj: Final = cast(AnthropicMessagesResponse, dict(rust_response)) - response_obj["_hidden_params"] = {"additional_headers": {"x-litellm-rust": "true"}} return response_obj @staticmethod @@ -2430,7 +2423,7 @@ class BaseLLMHTTPHandler: ) completion_stream = cast(AsyncIterator[bytes], FakeAnthropicMessagesStreamIterator(response=rust_response)) - hidden_params: Final = AnthropicMessagesStreamHiddenParams(additional_headers={"x-litellm-rust": "true"}) + hidden_params: Final = AnthropicMessagesStreamHiddenParams(additional_headers={}) return AnthropicMessagesStreamingResponse( completion_stream=completion_stream, hidden_params=hidden_params, diff --git a/litellm/rust_bridge/messages.py b/litellm/rust_bridge/messages.py index 40d0ddf622b..a4890a626d8 100644 --- a/litellm/rust_bridge/messages.py +++ b/litellm/rust_bridge/messages.py @@ -3,11 +3,19 @@ from __future__ import annotations from collections.abc import Awaitable -from dataclasses import dataclass -from typing import Final, Protocol, cast +from typing import Final, Protocol import httpx +from litellm.rust_bridge.bindings import NativeBinding +from litellm.rust_bridge.runtime import ( + BridgeErrorContext, + FallbackMode, + ainvoke, + async_none, + identity, + invoke, +) from litellm.rust_bridge.timeouts import timeout_to_seconds @@ -39,20 +47,15 @@ class RustAmessages(Protocol): raise NotImplementedError +_MESSAGES: Final = NativeBinding[RustMessages]("messages") +_AMESSAGES: Final = NativeBinding[RustAmessages]("amessages") + + class _Unset: pass -_UNSET: Final[_Unset] = _Unset() - - -@dataclass(slots=True) -class _RustMessagesState: - messages: RustMessages | None = None - amessages: RustAmessages | None = None - - -_STATE: Final[_RustMessagesState] = _RustMessagesState() +_UNSET: Final = _Unset() def set_rust_messages( @@ -61,31 +64,25 @@ def set_rust_messages( amessages: RustAmessages | None | _Unset = _UNSET, ) -> None: if not isinstance(messages, _Unset): - _STATE.messages = messages + _MESSAGES.reset() if messages is None else _MESSAGES.override(messages) if not isinstance(amessages, _Unset): - _STATE.amessages = amessages + _AMESSAGES.reset() if amessages is None else _AMESSAGES.override(amessages) def load_rust_messages() -> RustMessages | None: - if _STATE.messages is not None: - return _STATE.messages - from litellm.rust_bridge import get_native_bridge - - native_bridge: Final = get_native_bridge() - if native_bridge is None: - return None - return cast(RustMessages, getattr(native_bridge, "messages", None)) + return _MESSAGES.load() def load_rust_amessages() -> RustAmessages | None: - if _STATE.amessages is not None: - return _STATE.amessages - from litellm.rust_bridge import get_native_bridge + return _AMESSAGES.load() - native_bridge: Final = get_native_bridge() - if native_bridge is None: - return None - return cast(RustAmessages, getattr(native_bridge, "amessages", None)) + +def _context(model: str, custom_llm_provider: str | None) -> BridgeErrorContext: + return BridgeErrorContext( + route="messages", + provider=custom_llm_provider or "anthropic", + model=model, + ) def messages( @@ -98,17 +95,25 @@ def messages( extra_headers: dict[str, object] | None, timeout: float | httpx.Timeout | None, ) -> dict[str, object] | None: - rust_messages: Final = load_rust_messages() - if rust_messages is None: - return None - return rust_messages( - model=model, - body=body, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - timeout_seconds=timeout_to_seconds(timeout), + native: Final = load_rust_messages() + return invoke( + native_call=( + None + if native is None + else lambda: native( + model=model, + body=body, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + timeout_seconds=timeout_to_seconds(timeout), + ) + ), + fallback=lambda: None, + adapt=identity, + mode=FallbackMode.PYTHON, + context=_context(model, custom_llm_provider), ) @@ -122,15 +127,23 @@ async def amessages( extra_headers: dict[str, object] | None, timeout: float | httpx.Timeout | None, ) -> dict[str, object] | None: - rust_amessages: Final = load_rust_amessages() - if rust_amessages is None: - return None - return await rust_amessages( - model=model, - body=body, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - timeout_seconds=timeout_to_seconds(timeout), + native: Final = load_rust_amessages() + return await ainvoke( + native_call=( + None + if native is None + else lambda: native( + model=model, + body=body, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + timeout_seconds=timeout_to_seconds(timeout), + ) + ), + fallback=async_none, + adapt=identity, + mode=FallbackMode.PYTHON, + context=_context(model, custom_llm_provider), ) diff --git a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py index 293f75b7592..90a5c8c35f3 100644 --- a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py +++ b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py @@ -1,12 +1,14 @@ """Tests for the optional Rust-backed Anthropic Messages path.""" import importlib +from types import ModuleType from typing import cast import httpx import pytest import litellm +from litellm.exceptions import APIError from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.rust_bridge import configuration from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -100,12 +102,28 @@ class ExplodingAsyncMessages: class RaisingAsyncMessages: - def __init__(self) -> None: + def __init__(self, error: Exception) -> None: self.calls = 0 + self.error = error async def __call__(self, **kwargs: object) -> dict[str, object]: self.calls += 1 - raise RuntimeError("upstream request failed with status 400: bad request") + raise self.error + + +class FakeBridgeDeclined(Exception): + pass + + +class FakeUpstreamError(Exception): + pass + + +def _install_fake_bridge_exceptions(monkeypatch) -> None: + native_bridge = ModuleType("_native") + native_bridge.RustBridgeDeclined = FakeBridgeDeclined + native_bridge.RustUpstreamError = FakeUpstreamError + monkeypatch.setattr(rust_bridge_loader, "_cached_bridge", native_bridge) @pytest.fixture(autouse=True) @@ -224,7 +242,7 @@ def _gate(**overrides): @pytest.mark.asyncio -async def test_gate_invokes_rust_and_marks_response_header(): +async def test_gate_invokes_rust(): bridge = RecordingAsyncMessages() litellm.use_litellm_rust(True, amessages=bridge) @@ -232,7 +250,6 @@ async def test_gate_invokes_rust_and_marks_response_header(): assert response is not None assert response["id"] == "msg_123" - assert response["_hidden_params"]["additional_headers"] == {"x-litellm-rust": "true"} call = bridge.calls[0] assert call["model"] == "claude-sonnet-4-5" assert call["body"] == REQUEST_BODY @@ -243,8 +260,9 @@ async def test_gate_invokes_rust_and_marks_response_header(): @pytest.mark.asyncio -async def test_gate_falls_back_to_python_when_bridge_raises(): - bridge = RaisingAsyncMessages() +async def test_gate_falls_back_only_when_bridge_declines(monkeypatch): + _install_fake_bridge_exceptions(monkeypatch) + bridge = RaisingAsyncMessages(FakeBridgeDeclined("unsupported request")) litellm.use_litellm_rust(True, amessages=bridge) response = await _gate() @@ -253,6 +271,45 @@ async def test_gate_falls_back_to_python_when_bridge_raises(): assert bridge.calls == 1 +@pytest.mark.asyncio +async def test_gate_surfaces_an_upstream_failure_without_fallback(monkeypatch): + _install_fake_bridge_exceptions(monkeypatch) + bridge = RaisingAsyncMessages(FakeUpstreamError(429, "429: rate limited")) + litellm.use_litellm_rust(True, amessages=bridge) + + with pytest.raises(APIError) as exc_info: + await _gate() + + assert exc_info.value.status_code == 429 + assert "429: rate limited" in str(exc_info.value) + assert bridge.calls == 1 + + +@pytest.mark.asyncio +async def test_gate_maps_statusless_upstream_failure_to_500_without_fallback(monkeypatch): + _install_fake_bridge_exceptions(monkeypatch) + bridge = RaisingAsyncMessages(FakeUpstreamError(0, "request timed out")) + litellm.use_litellm_rust(True, amessages=bridge) + + with pytest.raises(APIError) as exc_info: + await _gate() + + assert exc_info.value.status_code == 500 + assert "request timed out" in str(exc_info.value) + assert bridge.calls == 1 + + +@pytest.mark.asyncio +async def test_gate_reraises_an_unknown_bridge_failure(): + bridge = RaisingAsyncMessages(RuntimeError("unknown bridge failure")) + litellm.use_litellm_rust(True, amessages=bridge) + + with pytest.raises(RuntimeError, match="unknown bridge failure"): + await _gate() + + assert bridge.calls == 1 + + @pytest.mark.asyncio async def test_gate_skips_rust_when_flag_absent(): bridge = ExplodingAsyncMessages() @@ -301,7 +358,6 @@ async def test_gate_invokes_rust_for_native_anthropic_provider(): ) assert response is not None - assert response["_hidden_params"]["additional_headers"] == {"x-litellm-rust": "true"} assert bridge.calls[0]["custom_llm_provider"] == "anthropic" assert bridge.calls[0]["api_key"] == "sk-ant" @@ -370,7 +426,6 @@ async def test_gate_streams_through_rust_when_eligible_and_strips_stream_flag(): ) assert response is not None - assert response["_hidden_params"]["additional_headers"] == {"x-litellm-rust": "true"} assert "stream" not in bridge.calls[0]["body"] assert bridge.calls[0]["body"] == REQUEST_BODY @@ -380,7 +435,7 @@ async def test_fake_stream_wraps_rust_response_as_anthropic_sse(): response = cast(AnthropicMessagesResponse, dict(FAKE_MESSAGES_RESPONSE)) stream = BaseLLMHTTPHandler._rust_anthropic_messages_fake_stream(response) - assert stream._hidden_params["additional_headers"] == {"x-litellm-rust": "true"} + assert stream._hidden_params["additional_headers"] == {} chunks = [chunk async for chunk in stream] joined = b"".join(chunks) diff --git a/tests/test_litellm/rust_bridge/native_route_wheel_test.py b/tests/test_litellm/rust_bridge/native_route_wheel_test.py index 378194191e5..2789def7784 100644 --- a/tests/test_litellm/rust_bridge/native_route_wheel_test.py +++ b/tests/test_litellm/rust_bridge/native_route_wheel_test.py @@ -4,6 +4,7 @@ import asyncio import importlib.util import json import os +import shutil import signal import subprocess import sys @@ -170,6 +171,19 @@ def route_kwargs(route: str, api_base: str, outcome: str) -> dict[str, object]: raise AssertionError(f"unknown route: {route}") +def messages_wrapper_kwargs(api_base: str, outcome: str) -> dict[str, object]: + native_kwargs: Final = route_kwargs("messages", api_base, outcome) + return { + "model": native_kwargs["model"], + "body": native_kwargs["body"], + "api_key": native_kwargs["api_key"], + "api_base": native_kwargs["api_base"], + "custom_llm_provider": native_kwargs["custom_llm_provider"], + "extra_headers": native_kwargs["extra_headers"], + "timeout": native_kwargs["timeout_seconds"], + } + + def assert_success(route: str, response: object) -> None: if not isinstance(response, dict): raise TypeError(f"{route} returned {type(response).__name__}, expected dict") @@ -192,7 +206,7 @@ def success_value(route: str, response: dict[object, object]) -> object: def assert_rate_limit(native: object, route: str, error: BaseException) -> None: - if route == "chat_completions": + if route in ("chat_completions", "messages"): upstream_error: Final = native.RustUpstreamError if not isinstance(error, upstream_error) or error.args[0] != 429: raise AssertionError(f"{route} returned the wrong 429 error: {error!r}") @@ -247,6 +261,86 @@ def exercise_routes(native_path: Path, api_base: str) -> object: return native +def assert_packaged_native_loaded(wheel_root: Path) -> None: + from litellm.rust_bridge import get_native_bridge + + native: Final = get_native_bridge() + if native is None: + raise AssertionError("packaged native bridge was not loaded") + native_file: Final = getattr(native, "__file__", None) + if not isinstance(native_file, str): + raise AssertionError("packaged native bridge has no module path") + if wheel_root.resolve() not in Path(native_file).resolve().parents: + raise AssertionError(f"native bridge loaded outside the wheel: {native_file}") + + +async def exercise_packaged_messages(api_base: str) -> None: + from litellm.exceptions import APIError + from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler + from litellm.rust_bridge import messages as messages_bridge + from litellm.types.router import GenericLiteLLMParams + + response: Final = messages_bridge.messages(**messages_wrapper_kwargs(api_base, "success")) + assert_success("messages", response) + + responses: Final = await asyncio.wait_for( + asyncio.gather( + *( + messages_bridge.amessages(**messages_wrapper_kwargs(api_base, "success")) + for _ in range(32) + ) + ), + timeout=15, + ) + for concurrent_response in responses: + assert_success("messages", concurrent_response) + + declined_kwargs: Final = messages_wrapper_kwargs(api_base, "success") + declined_kwargs["custom_llm_provider"] = "openai" + if await messages_bridge.amessages(**declined_kwargs) is not None: + raise AssertionError("unsupported Messages request did not decline") + + params: Final = GenericLiteLLMParams(api_key="sk-native", rust=True) + success_kwargs: Final = route_kwargs("messages", api_base, "success") + gate_response: Final = await BaseLLMHTTPHandler._maybe_rust_anthropic_messages( + custom_llm_provider="anthropic", + litellm_params=params, + has_agentic_hook=False, + model=str(success_kwargs["model"]), + api_key="sk-native", + api_base=api_base, + headers=dict(success_kwargs["extra_headers"]), + request_body=dict(success_kwargs["body"]), + timeout=3.0, + ) + assert_success("messages", gate_response) + + rate_limit_kwargs: Final = route_kwargs("messages", api_base, "429") + try: + await BaseLLMHTTPHandler._maybe_rust_anthropic_messages( + custom_llm_provider="anthropic", + litellm_params=params, + has_agentic_hook=False, + model=str(rate_limit_kwargs["model"]), + api_key="sk-native", + api_base=api_base, + headers=dict(rate_limit_kwargs["extra_headers"]), + request_body=dict(rate_limit_kwargs["body"]), + timeout=3.0, + ) + except APIError as error: + if error.status_code != 429 or "native-rate-limit" not in str(error): + raise AssertionError(f"Messages gate returned the wrong upstream error: {error!r}") from error + else: + raise AssertionError("Messages gate retried or swallowed the native upstream error") + + +def exercise_packaged_python_bridge(wheel_root: Path, api_base: str) -> int: + assert_packaged_native_loaded(wheel_root) + asyncio.run(exercise_packaged_messages(api_base)) + return 0 + + def exercise_signal(native: object, api_base: str) -> int: try: native.messages( @@ -293,6 +387,40 @@ def verify_sigint(native_path: Path, api_base: str) -> None: process.wait(timeout=5) +def verify_packaged_python_bridge(wheel: Path, wheel_root: Path, api_base: str) -> None: + uv: Final = shutil.which("uv") + if uv is None: + raise AssertionError("uv is required to test the packaged Python bridge") + environment: Final = {key: value for key, value in os.environ.items() if key != "ANTHROPIC_API_KEY"} | { + "PYTHONPATH": str(wheel_root) + } + result: Final = subprocess.run( + ( + uv, + "run", + "--isolated", + "--with", + str(wheel.resolve()), + "python", + __file__, + "bridge-child", + str(wheel_root), + api_base, + ), + cwd=wheel_root, + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if result.returncode != 0: + raise AssertionError( + f"packaged Python bridge failed with status {result.returncode}" + f"\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + + def verify_wheel(wheel: Path) -> int: with tempfile.TemporaryDirectory() as temporary_directory, zipfile.ZipFile(wheel) as archive: wheel_root: Final = Path(temporary_directory) @@ -318,6 +446,7 @@ def verify_wheel(wheel: Path) -> int: api_base: Final = f"http://127.0.0.1:{server.server_address[1]}" try: verify_sigint(native_path, api_base) + verify_packaged_python_bridge(wheel, wheel_root, api_base) finally: server.shutdown() server.server_close() @@ -331,6 +460,8 @@ def main() -> int: if len(sys.argv) == 4 and sys.argv[1] == "child": native: Final = exercise_routes(Path(sys.argv[2]), sys.argv[3]) return exercise_signal(native, sys.argv[3]) + if len(sys.argv) == 4 and sys.argv[1] == "bridge-child": + return exercise_packaged_python_bridge(Path(sys.argv[2]), sys.argv[3]) sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") return 2 From 5e46874430f043c01dc1c7a91bb1105647f6600f Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 06:44:45 -0700 Subject: [PATCH 59/67] refactor(python-bridge): centralize native route invocation --- litellm/llms/anthropic/chat/handler.py | 122 ++++---- litellm/llms/bedrock/chat/converse_handler.py | 191 ++++++------ litellm/rust_bridge/bindings.py | 18 +- litellm/rust_bridge/chat_completions.py | 290 +++++++++--------- litellm/rust_bridge/configuration.py | 45 ++- litellm/rust_bridge/messages.py | 19 +- litellm/rust_bridge/ocr.py | 113 +++---- litellm/rust_bridge/responses_websocket.py | 79 ++--- litellm/rust_bridge/transcription.py | 127 ++++---- .../test_rust_bridge_messages.py | 14 +- .../chat/test_anthropic_chat_handler.py | 11 +- .../chat/test_bedrock_converse_handler.py | 15 +- tests/test_litellm/ocr/test_rust_bridge.py | 111 ++++--- .../responses/test_rust_bridge_websocket.py | 5 +- .../rust_bridge/test_chat_completions.py | 194 ++++++------ .../test_audio_transcription_rust_bridge.py | 3 +- 16 files changed, 667 insertions(+), 690 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index c82be07a5c5..b5e18bd8e7e 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -4,7 +4,7 @@ Calling + translation logic for anthropic's `/v1/messages` endpoint import copy import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal, Union, cast import httpx @@ -380,7 +380,10 @@ class AnthropicChatCompletion(BaseLLM): request_data: Final = config.transform_request( model=model, messages=messages, - optional_params={**optional_params, "is_vertex_request": is_vertex_request}, + optional_params={ # mutable-ok: provider transforms require a plain mutable request dict + **optional_params, + "is_vertex_request": is_vertex_request, + }, litellm_params=litellm_params, headers=headers, ) @@ -390,6 +393,46 @@ class AnthropicChatCompletion(BaseLLM): provider=custom_llm_provider, ) + def sync_python_path( + request_headers: Mapping[str, object], request_data: Mapping[str, object] + ) -> ModelResponse: + request_client: Final = ( + _get_httpx_client(params={"timeout": timeout}) # mutable-ok: HTTP client factory requires a dict + if client is None or not isinstance(client, HTTPHandler) + else client + ) + try: + response: Final = request_client.post( + api_base, + headers=dict(request_headers), # mutable-ok: HTTP client requires mutable headers + data=json.dumps(request_data), + timeout=timeout, + logging_obj=logging_obj, + ) + except Exception as error: # noqa: BLE001 # provider exceptions are normalized below + status_code: Final = getattr(error, "status_code", 500) + error_headers = getattr(error, "headers", None) + error_text = getattr(error, "text", str(error)) + error_response: Final = getattr(error, "response", None) + if error_headers is None and error_response: + error_headers = getattr(error_response, "headers", None) + if error_response and hasattr(error_response, "text"): + error_text = getattr(error_response, "text", error_text) + raise AnthropicError(message=error_text, status_code=status_code, headers=error_headers) + return config.transform_response( + model=model, + raw_response=response, + model_response=model_response, + logging_obj=logging_obj, + api_key=api_key, + request_data=dict(request_data), # mutable-ok: response transform mutates request metadata + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + json_mode=json_mode, + ) + # The Rust core owns the whole call for the subset it accepts, so ask # before transforming: whichever path runs emits pre_call exactly once. # `get_config` merges the class-level defaults (Anthropic's required @@ -466,7 +509,12 @@ class AnthropicChatCompletion(BaseLLM): on_response=log_rust_post_call, python_fallback=python_fallback, ) - rust_response: Final = rust_chat_completions_bridge.chat_completions( + + def sync_python_fallback() -> ModelResponse: + fallback_headers, fallback_data = build_request() + return sync_python_path(fallback_headers, fallback_data) + + return rust_chat_completions_bridge.chat_completions_or_fallback( model=model, messages=messages, optional_params=rust_optional_params, @@ -477,26 +525,21 @@ class AnthropicChatCompletion(BaseLLM): extra_headers=headers, timeout=timeout, on_response=log_rust_post_call, + python_fallback=sync_python_fallback, ) - if rust_response is not None: - return rust_response headers, data = build_request() ## LOGGING - # Reaching here with `serves_via_rust` set means the Rust attempt - # declined at call time, before the provider was called, and already - # logged this request. That is the same attempt continuing. - if not serves_via_rust: - logging_obj.pre_call( - input=messages, - api_key=api_key, - additional_args={ - "complete_input_dict": data, - "api_base": api_base, - "headers": headers, - }, - ) + logging_obj.pre_call( + input=messages, + api_key=api_key, + additional_args={ # mutable-ok: logging callback contract requires a mutable dict + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) print_verbose(f"_is_function_call: {_is_function_call}") if acompletion is True: if ( @@ -579,48 +622,7 @@ class AnthropicChatCompletion(BaseLLM): _response_headers=process_anthropic_headers(headers), ) - else: - if client is None or not isinstance(client, HTTPHandler): - client = _get_httpx_client(params={"timeout": timeout}) - else: - client = client - - try: - response: Final = client.post( - api_base, - headers=headers, - data=json.dumps(data), - timeout=timeout, - logging_obj=logging_obj, - ) - except Exception as e: - status_code: Final = getattr(e, "status_code", 500) - error_headers = getattr(e, "headers", None) - error_text = getattr(e, "text", str(e)) - error_response: Final[object] = getattr(e, "response", None) - if error_headers is None and error_response: - error_headers = getattr(error_response, "headers", None) - if error_response and hasattr(error_response, "text"): - error_text = getattr(error_response, "text", error_text) - raise AnthropicError( - message=error_text, - status_code=status_code, - headers=error_headers, - ) - - return config.transform_response( - model=model, - raw_response=response, - model_response=model_response, - logging_obj=logging_obj, - api_key=api_key, - request_data=data, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params, - encoding=encoding, - json_mode=json_mode, - ) + return sync_python_path(headers, data) def embedding(self): # logic for parsing in - calling - parsing out model embedding calls diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 7d5f99ca893..7d6a4d3e6a1 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -383,6 +383,91 @@ class BedrockConverseLLM(BaseAWSLLM): # Filter beta headers in HTTP headers before making the request headers = update_headers_with_filtered_beta(headers=headers, provider="bedrock_converse") + def sync_python_path(*, skip_pre_call_logging: bool) -> ModelResponse | CustomStreamWrapper: + request_data: Final = litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=extra_headers, + ) + serialized_data: Final = json.dumps(request_data) + prepped: Final = self.get_request_headers( + credentials=credentials, + aws_region_name=aws_region_name, + extra_headers=extra_headers, + endpoint_url=proxy_endpoint_url, + data=serialized_data, + headers=headers, + api_key=api_key, + ) + if not skip_pre_call_logging: + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ # mutable-ok: logging callback contract requires a mutable dict + "complete_input_dict": serialized_data, + "api_base": proxy_endpoint_url, + "headers": prepped.headers, + }, + ) + request_timeout: Final = ( + httpx.Timeout(timeout) if isinstance(timeout, float) or isinstance(timeout, int) else timeout + ) + request_client: Final = ( + _get_httpx_client( # mutable-ok: HTTP client factory requires a mutable options dict + {"timeout": request_timeout} if request_timeout is not None else {} + ) + if client is None or isinstance(client, AsyncHTTPHandler) + else client + ) + if stream is not None and stream is True: + completion_stream, response_headers = make_sync_call( + client=request_client if isinstance(request_client, HTTPHandler) else None, + api_base=proxy_endpoint_url, + headers=prepped.headers, + data=serialized_data, + model=model, + messages=messages, + logging_obj=logging_obj, + json_mode=json_mode, + fake_stream=fake_stream, + stream_chunk_size=stream_chunk_size, + ) + return CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + _response_headers=response_headers, + ) + try: + response: Final = request_client.post( + url=proxy_endpoint_url, + headers=prepped.headers, + data=serialized_data, + logging_obj=logging_obj, + ) + response.raise_for_status() + except httpx.HTTPStatusError as error: + raise BedrockError(status_code=error.response.status_code, message=error.response.text) + except httpx.TimeoutException: + raise BedrockError(status_code=408, message="Timeout error occurred.") + transformed: Final = litellm.AmazonConverseConfig()._transform_response( + model=model, + response=response, + model_response=model_response, + stream=stream if isinstance(stream, bool) else False, + logging_obj=logging_obj, + api_key="", + data=serialized_data, + messages=messages, + optional_params=optional_params, + encoding=encoding, + ) + transformed.set_provider_response_headers(response.headers) + return transformed + # The Rust core owns the whole call for the subset it accepts. Ask # before transforming so whichever path runs emits pre_call once, and # hand down the credentials, region and endpoint this handler already @@ -449,7 +534,7 @@ class BedrockConverseLLM(BaseAWSLLM): skip_pre_call_logging=True, ), ) - rust_response: Final = rust_chat_completions_bridge.chat_completions( + return rust_chat_completions_bridge.chat_completions_or_fallback( model=model, messages=messages, optional_params=rust_optional_params, @@ -460,9 +545,8 @@ class BedrockConverseLLM(BaseAWSLLM): extra_headers=headers, timeout=timeout, on_response=log_rust_post_call, + python_fallback=lambda: sync_python_path(skip_pre_call_logging=True), ) - if rust_response is not None: - return rust_response ### ROUTING (ASYNC, STREAMING, SYNC) if acompletion: @@ -508,103 +592,4 @@ class BedrockConverseLLM(BaseAWSLLM): api_key=api_key, ) - ## TRANSFORMATION ## - - _data: Final = litellm.AmazonConverseConfig()._transform_request( - model=model, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params, - headers=extra_headers, - ) - data: Final = json.dumps(_data) - - prepped: Final = self.get_request_headers( - credentials=credentials, - aws_region_name=aws_region_name, - extra_headers=extra_headers, - endpoint_url=proxy_endpoint_url, - data=data, - headers=headers, - api_key=api_key, - ) - - ## LOGGING - # Reaching here with `serves_via_rust` set means the synchronous Rust - # attempt declined at call time, before the provider was called, and - # already logged this request. That is the same attempt continuing. - # The asynchronous branch above returns before this point, and hands - # its own fallback `skip_pre_call_logging=True` for the same reason. - if not serves_via_rust: - logging_obj.pre_call( - input=messages, - api_key="", - additional_args={ - "complete_input_dict": data, - "api_base": proxy_endpoint_url, - "headers": prepped.headers, - }, - ) - if client is None or isinstance(client, AsyncHTTPHandler): - _params: Final = {} - if timeout is not None: - if isinstance(timeout, float) or isinstance(timeout, int): - timeout = httpx.Timeout(timeout) - _params["timeout"] = timeout - client = _get_httpx_client(_params) - else: - client = client - - if stream is not None and stream is True: - completion_stream, response_headers = make_sync_call( - client=(client if client is not None and isinstance(client, HTTPHandler) else None), - api_base=proxy_endpoint_url, - headers=prepped.headers, - data=data, - model=model, - messages=messages, - logging_obj=logging_obj, - json_mode=json_mode, - fake_stream=fake_stream, - stream_chunk_size=stream_chunk_size, - ) - streaming_response: Final = CustomStreamWrapper( - completion_stream=completion_stream, - model=model, - custom_llm_provider="bedrock", - logging_obj=logging_obj, - _response_headers=response_headers, - ) - - return streaming_response - - ### COMPLETION - - try: - response: Final = client.post( - url=proxy_endpoint_url, - headers=prepped.headers, - data=data, - logging_obj=logging_obj, - ) - response.raise_for_status() - except httpx.HTTPStatusError as err: - error_code: Final = err.response.status_code - raise BedrockError(status_code=error_code, message=err.response.text) - except httpx.TimeoutException: - raise BedrockError(status_code=408, message="Timeout error occurred.") - - sync_transformed_response: Final = litellm.AmazonConverseConfig()._transform_response( - model=model, - response=response, - model_response=model_response, - stream=stream if isinstance(stream, bool) else False, - logging_obj=logging_obj, - api_key="", - data=data, - messages=messages, - optional_params=optional_params, - encoding=encoding, - ) - sync_transformed_response.set_provider_response_headers(response.headers) - return sync_transformed_response + return sync_python_path(skip_pre_call_logging=False) diff --git a/litellm/rust_bridge/bindings.py b/litellm/rust_bridge/bindings.py index a4c529f7deb..25b0e1a17e7 100644 --- a/litellm/rust_bridge/bindings.py +++ b/litellm/rust_bridge/bindings.py @@ -7,11 +7,11 @@ from litellm.rust_bridge.loader import get_native_bridge BindingT = TypeVar("BindingT") -class _Unset: +class Unset: pass -_UNSET: Final = _Unset() +UNSET: Final = Unset() class NativeBinding(Generic[BindingT]): @@ -19,10 +19,10 @@ class NativeBinding(Generic[BindingT]): def __init__(self, attribute: str) -> None: self._attribute: Final = attribute - self._override: BindingT | None | _Unset = _UNSET + self._override: BindingT | None | Unset = UNSET def load(self) -> BindingT | None: - if not isinstance(self._override, _Unset): + if not isinstance(self._override, Unset): return self._override native: Final = get_native_bridge() if native is None: @@ -33,7 +33,15 @@ class NativeBinding(Generic[BindingT]): self._override = value def reset(self) -> None: - self._override = _UNSET + self._override = UNSET + + def update(self, value: BindingT | None | Unset) -> None: + if isinstance(value, Unset): + return + if value is None: + self.reset() + else: + self.override(value) def native_exception_types() -> tuple[type[BaseException], type[BaseException]] | None: diff --git a/litellm/rust_bridge/chat_completions.py b/litellm/rust_bridge/chat_completions.py index c599667ab17..efdb6eaa9c5 100644 --- a/litellm/rust_bridge/chat_completions.py +++ b/litellm/rust_bridge/chat_completions.py @@ -14,20 +14,29 @@ from __future__ import annotations import json from collections.abc import Awaitable, Callable, Mapping, Sequence -from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Protocol import httpx from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger -from litellm.exceptions import APIError from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, ) from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned +from litellm.rust_bridge.bindings import UNSET, NativeBinding, Unset from litellm.rust_bridge.configuration import rust_enabled -from litellm.rust_bridge.loader import get_native_bridge +from litellm.rust_bridge.runtime import ( + BridgeErrorContext, + FallbackMode, + RustDeclined, + RustHandled, + aattempt, + ainvoke, + attempt, + identity, + invoke, +) from litellm.rust_bridge.timeouts import timeout_to_seconds from litellm.types.utils import ModelResponse @@ -42,9 +51,6 @@ RUST_CHAT_COMPLETIONS_PROVIDERS: Final = frozenset({"anthropic", "bedrock"}) # rather than narrowing an unparameterized `Mapping` and typing the result Any. _LITELLM_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -RUST_RESPONSE_HEADER: Final = "x-litellm-rust" - - class RustChatCompletions(Protocol): def __call__( self, @@ -126,67 +132,34 @@ def response_logger( return log -class _Unset: - pass - - -_UNSET: Final[_Unset] = _Unset() - - -@dataclass(slots=True) -class _RustChatCompletionsState: - chat_completions: RustChatCompletions | None = None - achat_completions: RustAchatCompletions | None = None - decline: RustChatCompletionsDecline | None = None - - -_STATE: Final[_RustChatCompletionsState] = _RustChatCompletionsState() +_CHAT_COMPLETIONS: Final = NativeBinding[RustChatCompletions]("chat_completions") +_ACHAT_COMPLETIONS: Final = NativeBinding[RustAchatCompletions]("achat_completions") +_DECLINE: Final = NativeBinding[RustChatCompletionsDecline]("chat_completions_decline") def set_rust_chat_completions( *, - chat_completions: RustChatCompletions | None | _Unset = _UNSET, - achat_completions: RustAchatCompletions | None | _Unset = _UNSET, - decline: RustChatCompletionsDecline | None | _Unset = _UNSET, + chat_completions: RustChatCompletions | None | Unset = UNSET, + achat_completions: RustAchatCompletions | None | Unset = UNSET, + decline: RustChatCompletionsDecline | None | Unset = UNSET, ) -> None: """Inject the native callables, so tests can supply a double instead of patching module attributes.""" - if not isinstance(chat_completions, _Unset): - _STATE.chat_completions = chat_completions - if not isinstance(achat_completions, _Unset): - _STATE.achat_completions = achat_completions - if not isinstance(decline, _Unset): - _STATE.decline = decline + _CHAT_COMPLETIONS.update(chat_completions) + _ACHAT_COMPLETIONS.update(achat_completions) + _DECLINE.update(decline) def load_rust_chat_completions() -> RustChatCompletions | None: - if _STATE.chat_completions is not None: - return _STATE.chat_completions - native_bridge: Final = get_native_bridge() - if native_bridge is None: - return None - loaded: RustChatCompletions | None = getattr(native_bridge, "chat_completions", None) - return loaded + return _CHAT_COMPLETIONS.load() def load_rust_achat_completions() -> RustAchatCompletions | None: - if _STATE.achat_completions is not None: - return _STATE.achat_completions - native_bridge: Final = get_native_bridge() - if native_bridge is None: - return None - loaded: RustAchatCompletions | None = getattr(native_bridge, "achat_completions", None) - return loaded + return _ACHAT_COMPLETIONS.load() def _load_rust_decline() -> RustChatCompletionsDecline | None: - if _STATE.decline is not None: - return _STATE.decline - native_bridge: Final = get_native_bridge() - if native_bridge is None: - return None - loaded: RustChatCompletionsDecline | None = getattr(native_bridge, "chat_completions_decline", None) - return loaded + return _DECLINE.load() def _anthropic_user_id_reaches_the_body(litellm_params: Mapping[str, object] | None) -> bool: @@ -256,76 +229,35 @@ def rust_chat_completions_accepts( decline: Final = _load_rust_decline() if decline is None: return False - try: - reason: Final = decline( + gate_result: Final = attempt( + native_call=lambda: decline( model=model, messages=messages, optional_params=optional_params, custom_llm_provider=custom_llm_provider, - ) - except Exception as rust_error: # noqa: BLE001 # rollout-safety fallback: any Rust bridge failure must fall back to the Python path + ), + adapt=identity, + context=BridgeErrorContext( + route="chat completions capability check", + provider=custom_llm_provider or "", + model=model, + ), + ) + if isinstance(gate_result, RustDeclined): verbose_logger.debug( - "Rust chat completions gate raised %s; staying on the Python path", - type(rust_error).__name__, + "Rust chat completions declined (%s); using the Python path", + gate_result.reason, ) return False + if not isinstance(gate_result, RustHandled): + return False + reason: Final = gate_result.value if reason is not None: verbose_logger.debug("Rust chat completions declined (%s); using the Python path", reason) return False return True -def _rust_bridge_exceptions() -> tuple[type[BaseException], type[BaseException]] | None: - """`(declined, upstream_failed)` from the native module, or None when absent.""" - native_bridge: Final = get_native_bridge() - if native_bridge is None: - return None - declined: Final = getattr(native_bridge, "RustBridgeDeclined", None) - upstream: Final = getattr(native_bridge, "RustUpstreamError", None) - if declined is None or upstream is None: - return None - return declined, upstream - - -def _reraise_or_decline( - rust_error: BaseException, - *, - model: str, - custom_llm_provider: str | None, -) -> None: - """Re-raise a failure the provider already saw, or return so the caller declines. - - A request that never reached the provider is safe to serve on the Python - path. One that did is not: the provider has already done the work, so a - second attempt bills for it twice. Those surface as an `APIError` carrying - the upstream status, which LiteLLM's exception mapping already understands. - """ - exceptions: Final = _rust_bridge_exceptions() - if exceptions is None: - verbose_logger.debug( - "Rust chat completions bridge raised %s; falling back to Python path", - type(rust_error).__name__, - ) - return - declined, upstream_failed = exceptions - if isinstance(rust_error, upstream_failed): - args: Final = rust_error.args - status: Final = args[0] if args else 0 - message: Final = args[1] if len(args) > 1 else "" - raise APIError( - status_code=int(status) or 500, - message=f"litellm rust chat completions: {message}", - llm_provider=custom_llm_provider or "", - model=model, - ) - if not isinstance(rust_error, declined): - raise rust_error - verbose_logger.debug( - "Rust chat completions declined before calling the provider (%s); using the Python path", - rust_error, - ) - - def _build_model_response( rust_response: Mapping[str, object], model_response: ModelResponse, @@ -333,7 +265,6 @@ def _build_model_response( built: Final = convert_to_model_response_object( response_object=dict(rust_response), # mutable-ok: the converter takes a real dict and rewrites it model_response_object=model_response, - hidden_params={"additional_headers": {RUST_RESPONSE_HEADER: "true"}}, # mutable-ok: rewritten by the converter ) if not isinstance(built, ModelResponse): raise TypeError(f"expected a ModelResponse from the rust path, got {type(built).__name__}") @@ -354,10 +285,10 @@ def chat_completions( on_response: ResponseObserver, ) -> ModelResponse | None: rust_chat_completions: Final = load_rust_chat_completions() - if rust_chat_completions is None: - return None - try: - rust_response: Final = rust_chat_completions( + native_call: Final = ( + None + if rust_chat_completions is None + else lambda: rust_chat_completions( model=model, messages=messages, optional_params=optional_params, @@ -367,11 +298,18 @@ def chat_completions( extra_headers=extra_headers, timeout_seconds=timeout_to_seconds(timeout), ) - except Exception as rust_error: # noqa: BLE001 # rollout safety: the helper re-raises anything the provider already saw - _reraise_or_decline(rust_error, model=model, custom_llm_provider=custom_llm_provider) - return None - on_response(rust_response) - return _build_model_response(rust_response, model_response) + ) + + def adapt(rust_response: Mapping[str, object]) -> ModelResponse: + on_response(rust_response) + return _build_model_response(rust_response, model_response) + + result: Final = attempt( + native_call=native_call, + adapt=adapt, + context=BridgeErrorContext(route="chat completions", provider=custom_llm_provider or "", model=model), + ) + return result.value if isinstance(result, RustHandled) else None async def achat_completions( @@ -388,10 +326,10 @@ async def achat_completions( on_response: ResponseObserver, ) -> ModelResponse | None: rust_achat_completions: Final = load_rust_achat_completions() - if rust_achat_completions is None: - return None - try: - rust_response: Final = await rust_achat_completions( + native_call: Final = ( + None + if rust_achat_completions is None + else lambda: rust_achat_completions( model=model, messages=messages, optional_params=optional_params, @@ -401,11 +339,61 @@ async def achat_completions( extra_headers=extra_headers, timeout_seconds=timeout_to_seconds(timeout), ) - except Exception as rust_error: # noqa: BLE001 # rollout safety: the helper re-raises anything the provider already saw - _reraise_or_decline(rust_error, model=model, custom_llm_provider=custom_llm_provider) - return None - on_response(rust_response) - return _build_model_response(rust_response, model_response) + ) + + def adapt(rust_response: Mapping[str, object]) -> ModelResponse: + on_response(rust_response) + return _build_model_response(rust_response, model_response) + + result: Final = await aattempt( + native_call=native_call, + adapt=adapt, + context=BridgeErrorContext(route="chat completions", provider=custom_llm_provider or "", model=model), + ) + return result.value if isinstance(result, RustHandled) else None + + +def chat_completions_or_fallback( + *, + model: str, + messages: Sequence[object], + optional_params: Mapping[str, object], + model_response: ModelResponse, + api_key: str | None, + api_base: str | None, + custom_llm_provider: str | None, + extra_headers: Mapping[str, object] | None, + timeout: float | httpx.Timeout | None, + on_response: ResponseObserver, + python_fallback: Callable[[], object], +) -> object: + rust_chat_completions: Final = load_rust_chat_completions() + native_call: Final = ( + None + if rust_chat_completions is None + else lambda: rust_chat_completions( + model=model, + messages=messages, + optional_params=optional_params, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + timeout_seconds=timeout_to_seconds(timeout), + ) + ) + + def adapt(rust_response: Mapping[str, object]) -> object: + on_response(rust_response) + return _build_model_response(rust_response, model_response) + + return invoke( + native_call=native_call, + fallback=python_fallback, + adapt=adapt, + mode=FallbackMode.PYTHON, + context=BridgeErrorContext(route="chat completions", provider=custom_llm_provider or "", model=model), + ) async def achat_completions_or_fallback( @@ -422,26 +410,30 @@ async def achat_completions_or_fallback( on_response: ResponseObserver, python_fallback: Callable[[], Awaitable[object]], ) -> object: - """Await the Rust path, falling back to the caller's own Python path when - the bridge is unavailable or the call fails. - - The caller supplies the fallback, so the bridge stays free of provider - dispatch. This exists because a caller that dispatches asynchronously has - already returned a coroutine by the time a Rust failure surfaces, and so - cannot fall back on its own. - """ - response: Final = await achat_completions( - model=model, - messages=messages, - optional_params=optional_params, - model_response=model_response, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - timeout=timeout, - on_response=on_response, + rust_achat_completions: Final = load_rust_achat_completions() + native_call: Final = ( + None + if rust_achat_completions is None + else lambda: rust_achat_completions( + model=model, + messages=messages, + optional_params=optional_params, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + timeout_seconds=timeout_to_seconds(timeout), + ) + ) + + def adapt(rust_response: Mapping[str, object]) -> object: + on_response(rust_response) + return _build_model_response(rust_response, model_response) + + return await ainvoke( + native_call=native_call, + fallback=python_fallback, + adapt=adapt, + mode=FallbackMode.PYTHON, + context=BridgeErrorContext(route="chat completions", provider=custom_llm_provider or "", model=model), ) - if response is not None: - return response - return await python_fallback() diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index a8eecd022d6..8ae208c836a 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -4,6 +4,8 @@ import os import warnings from typing import TYPE_CHECKING, Final +from litellm.rust_bridge.bindings import UNSET, Unset + if TYPE_CHECKING: from litellm.rust_bridge.messages import RustAmessages, RustMessages from litellm.rust_bridge.ocr import RustAocr, RustOcr @@ -17,13 +19,6 @@ _GLOBAL_ENV_NAME: Final = "LITELLM_RUST" _LEGACY_OCR_ENV_NAME: Final = "LITELLM_USE_RUST_OCR" -class _Unset: - pass - - -_UNSET: Final = _Unset() - - class _RustConfiguration: def __init__(self) -> None: self.override: bool | None = None @@ -107,13 +102,13 @@ def reset_rust_configuration() -> None: def use_litellm_rust( enabled: bool = True, *, - ocr: RustOcr | None | _Unset = _UNSET, - aocr: RustAocr | None | _Unset = _UNSET, - messages: RustMessages | None | _Unset = _UNSET, - amessages: RustAmessages | None | _Unset = _UNSET, - responses_websocket: type[RustResponsesWebSocketConnection] | None | _Unset = _UNSET, - transcription: RustTranscription | None | _Unset = _UNSET, - atranscription: RustAtranscription | None | _Unset = _UNSET, + ocr: RustOcr | None | Unset = UNSET, + aocr: RustAocr | None | Unset = UNSET, + messages: RustMessages | None | Unset = UNSET, + amessages: RustAmessages | None | Unset = UNSET, + responses_websocket: type[RustResponsesWebSocketConnection] | None | Unset = UNSET, + transcription: RustTranscription | None | Unset = UNSET, + atranscription: RustAtranscription | None | Unset = UNSET, ) -> None: """Set the process override for optional Rust paths. @@ -121,7 +116,7 @@ def use_litellm_rust( """ _CONFIGURATION.override = enabled bindings: Final = (ocr, aocr, messages, amessages, responses_websocket, transcription, atranscription) - if all(isinstance(binding, _Unset) for binding in bindings): + if all(isinstance(binding, Unset) for binding in bindings): return warnings.warn( "Injecting Rust bridge implementations through use_litellm_rust() is deprecated; " @@ -130,28 +125,28 @@ def use_litellm_rust( stacklevel=2, ) - if not isinstance(ocr, _Unset) or not isinstance(aocr, _Unset): + if not isinstance(ocr, Unset) or not isinstance(aocr, Unset): from litellm.rust_bridge.ocr import set_rust_ocr - if not isinstance(ocr, _Unset): + if not isinstance(ocr, Unset): set_rust_ocr(ocr=ocr) - if not isinstance(aocr, _Unset): + if not isinstance(aocr, Unset): set_rust_ocr(aocr=aocr) - if not isinstance(messages, _Unset) or not isinstance(amessages, _Unset): + if not isinstance(messages, Unset) or not isinstance(amessages, Unset): from litellm.rust_bridge.messages import set_rust_messages - if not isinstance(messages, _Unset): + if not isinstance(messages, Unset): set_rust_messages(messages=messages) - if not isinstance(amessages, _Unset): + if not isinstance(amessages, Unset): set_rust_messages(amessages=amessages) - if not isinstance(responses_websocket, _Unset): + if not isinstance(responses_websocket, Unset): from litellm.rust_bridge.responses_websocket import set_rust_responses_websocket set_rust_responses_websocket(connection=responses_websocket) - if not isinstance(transcription, _Unset) or not isinstance(atranscription, _Unset): + if not isinstance(transcription, Unset) or not isinstance(atranscription, Unset): from litellm.rust_bridge.transcription import configure_rust_transcription - if not isinstance(transcription, _Unset): + if not isinstance(transcription, Unset): configure_rust_transcription(transcription=transcription) - if not isinstance(atranscription, _Unset): + if not isinstance(atranscription, Unset): configure_rust_transcription(atranscription=atranscription) diff --git a/litellm/rust_bridge/messages.py b/litellm/rust_bridge/messages.py index a4890a626d8..e0bdef20dfe 100644 --- a/litellm/rust_bridge/messages.py +++ b/litellm/rust_bridge/messages.py @@ -7,7 +7,7 @@ from typing import Final, Protocol import httpx -from litellm.rust_bridge.bindings import NativeBinding +from litellm.rust_bridge.bindings import UNSET, NativeBinding, Unset from litellm.rust_bridge.runtime import ( BridgeErrorContext, FallbackMode, @@ -51,22 +51,13 @@ _MESSAGES: Final = NativeBinding[RustMessages]("messages") _AMESSAGES: Final = NativeBinding[RustAmessages]("amessages") -class _Unset: - pass - - -_UNSET: Final = _Unset() - - def set_rust_messages( *, - messages: RustMessages | None | _Unset = _UNSET, - amessages: RustAmessages | None | _Unset = _UNSET, + messages: RustMessages | None | Unset = UNSET, + amessages: RustAmessages | None | Unset = UNSET, ) -> None: - if not isinstance(messages, _Unset): - _MESSAGES.reset() if messages is None else _MESSAGES.override(messages) - if not isinstance(amessages, _Unset): - _AMESSAGES.reset() if amessages is None else _AMESSAGES.override(amessages) + _MESSAGES.update(messages) + _AMESSAGES.update(amessages) def load_rust_messages() -> RustMessages | None: diff --git a/litellm/rust_bridge/ocr.py b/litellm/rust_bridge/ocr.py index cbe444d44a6..355254fb237 100644 --- a/litellm/rust_bridge/ocr.py +++ b/litellm/rust_bridge/ocr.py @@ -3,11 +3,20 @@ from __future__ import annotations from collections.abc import Awaitable -from typing import Final, Protocol, cast +from typing import Final, Protocol import httpx from litellm.rust_bridge import configuration as _configuration +from litellm.rust_bridge.bindings import UNSET, NativeBinding, Unset +from litellm.rust_bridge.runtime import ( + BridgeErrorContext, + FallbackMode, + ainvoke, + async_none, + identity, + invoke, +) from litellm.rust_bridge.timeouts import timeout_to_seconds as _timeout_to_seconds rust_ocr_enabled = _configuration.rust_ocr_enabled @@ -44,49 +53,25 @@ class RustAocr(Protocol): raise NotImplementedError -class _Unset: - pass - - -_UNSET: Final[_Unset] = _Unset() - - -_rust_ocr_impl: RustOcr | None = None -_rust_aocr_impl: RustAocr | None = None +_OCR: Final = NativeBinding[RustOcr]("ocr") +_AOCR: Final = NativeBinding[RustAocr]("aocr") def set_rust_ocr( *, - ocr: RustOcr | None | _Unset = _UNSET, - aocr: RustAocr | None | _Unset = _UNSET, + ocr: RustOcr | None | Unset = UNSET, + aocr: RustAocr | None | Unset = UNSET, ) -> None: - global _rust_ocr_impl, _rust_aocr_impl - if not isinstance(ocr, _Unset): - _rust_ocr_impl = ocr - if not isinstance(aocr, _Unset): - _rust_aocr_impl = aocr + _OCR.update(ocr) + _AOCR.update(aocr) def load_rust_ocr() -> RustOcr | None: - if _rust_ocr_impl is not None: - return _rust_ocr_impl - from litellm.rust_bridge import get_native_bridge - - native_bridge: Final = get_native_bridge() - if native_bridge is None: - return None - return cast(RustOcr, native_bridge.ocr) + return _OCR.load() def load_rust_aocr() -> RustAocr | None: - if _rust_aocr_impl is not None: - return _rust_aocr_impl - from litellm.rust_bridge import get_native_bridge - - native_bridge: Final = get_native_bridge() - if native_bridge is None: - return None - return cast(RustAocr, getattr(native_bridge, "aocr", None)) + return _AOCR.load() def ocr( @@ -101,17 +86,26 @@ def ocr( timeout: float | httpx.Timeout | None, ) -> dict[str, object] | None: rust_ocr: Final = load_rust_ocr() - if rust_ocr is None: - return None - return rust_ocr( - model=model, - document=document, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - optional_params=optional_params, - timeout_seconds=_timeout_to_seconds(timeout), + native_call: Final = ( + None + if rust_ocr is None + else lambda: rust_ocr( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + optional_params=optional_params, + timeout_seconds=_timeout_to_seconds(timeout), + ) + ) + return invoke( + native_call=native_call, + fallback=lambda: None, + adapt=identity, + mode=FallbackMode.PYTHON, + context=BridgeErrorContext(route="ocr", provider=custom_llm_provider or "", model=model), ) @@ -127,15 +121,24 @@ async def aocr( timeout: float | httpx.Timeout | None, ) -> dict[str, object] | None: rust_aocr: Final = load_rust_aocr() - if rust_aocr is None: - return None - return await rust_aocr( - model=model, - document=document, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - optional_params=optional_params, - timeout_seconds=_timeout_to_seconds(timeout), + native_call: Final = ( + None + if rust_aocr is None + else lambda: rust_aocr( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + optional_params=optional_params, + timeout_seconds=_timeout_to_seconds(timeout), + ) + ) + return await ainvoke( + native_call=native_call, + fallback=async_none, + adapt=identity, + mode=FallbackMode.PYTHON, + context=BridgeErrorContext(route="ocr", provider=custom_llm_provider or "", model=model), ) diff --git a/litellm/rust_bridge/responses_websocket.py b/litellm/rust_bridge/responses_websocket.py index 0634867af1c..4d505a757c9 100644 --- a/litellm/rust_bridge/responses_websocket.py +++ b/litellm/rust_bridge/responses_websocket.py @@ -2,13 +2,19 @@ from __future__ import annotations -from dataclasses import dataclass from typing import Final, Protocol import httpx from websockets.exceptions import ConnectionClosedOK -from litellm.rust_bridge.loader import get_native_bridge +from litellm.rust_bridge.bindings import UNSET, NativeBinding, Unset +from litellm.rust_bridge.runtime import ( + BridgeErrorContext, + FallbackMode, + acall, + ainvoke, + async_none, +) from litellm.rust_bridge.timeouts import timeout_to_seconds @@ -30,56 +36,46 @@ class RustResponsesWebSocketConnection(Protocol): ) -> RustResponsesWebSocket: ... -class _Unset: - pass - - -_UNSET: Final[_Unset] = _Unset() - - -@dataclass(slots=True) -class _RustResponsesWebSocketState: - connection: RustResponsesWebSocketConnection | None = None - - -_STATE: Final[_RustResponsesWebSocketState] = _RustResponsesWebSocketState() +_CONNECTION: Final = NativeBinding[type[RustResponsesWebSocketConnection]]( + "ResponsesWebSocketConnection" +) def set_rust_responses_websocket( *, - connection: RustResponsesWebSocketConnection | None | _Unset = _UNSET, + connection: type[RustResponsesWebSocketConnection] | None | Unset = UNSET, ) -> None: - if not isinstance(connection, _Unset): - _STATE.connection = connection + _CONNECTION.update(connection) -def load_rust_responses_websocket() -> RustResponsesWebSocketConnection | None: - if _STATE.connection is not None: - return _STATE.connection - native_bridge: Final = get_native_bridge() - if native_bridge is None: - return None - connection_type: Final[RustResponsesWebSocketConnection | None] = getattr( - native_bridge, "ResponsesWebSocketConnection", None - ) - return connection_type +def load_rust_responses_websocket() -> type[RustResponsesWebSocketConnection] | None: + return _CONNECTION.load() class _ConnectionAdapter: def __init__(self, connection: RustResponsesWebSocket): - self._connection: Final[RustResponsesWebSocket] = connection + self._connection: Final = connection async def send(self, text: str) -> None: - await self._connection.send_text(text) + await acall( + lambda: self._connection.send_text(text), + BridgeErrorContext(route="responses websocket", provider="openai", model=""), + ) async def recv(self) -> str: - message: Final = await self._connection.recv_text() + message: Final = await acall( + self._connection.recv_text, + BridgeErrorContext(route="responses websocket", provider="openai", model=""), + ) if message is None: raise ConnectionClosedOK(None, None) return message async def close(self) -> None: - await self._connection.close() + await acall( + self._connection.close, + BridgeErrorContext(route="responses websocket", provider="openai", model=""), + ) async def connect( @@ -89,14 +85,19 @@ async def connect( timeout: float | httpx.Timeout | None, ) -> _ConnectionAdapter | None: connection_type: Final = load_rust_responses_websocket() - if connection_type is None: - return None - try: - connection: Final = await connection_type.connect( + native_call: Final = ( + None + if connection_type is None + else lambda: connection_type.connect( url=url, headers=headers, timeout_seconds=timeout_to_seconds(timeout), ) - except Exception: # noqa: BLE001 # bridge failures must fall back to Python - return None - return _ConnectionAdapter(connection) + ) + return await ainvoke( + native_call=native_call, + fallback=async_none, + adapt=_ConnectionAdapter, + mode=FallbackMode.PYTHON, + context=BridgeErrorContext(route="responses websocket", provider="openai", model=""), + ) diff --git a/litellm/rust_bridge/transcription.py b/litellm/rust_bridge/transcription.py index 3d71f6f8a50..bf1e3fb60d8 100644 --- a/litellm/rust_bridge/transcription.py +++ b/litellm/rust_bridge/transcription.py @@ -1,11 +1,19 @@ from __future__ import annotations from collections.abc import Awaitable -from dataclasses import dataclass -from typing import Final, Protocol, cast +from typing import Final, Protocol import httpx +from litellm.rust_bridge.bindings import UNSET, NativeBinding, Unset +from litellm.rust_bridge.runtime import ( + BridgeErrorContext, + FallbackMode, + ainvoke, + async_none, + identity, + invoke, +) from litellm.rust_bridge.timeouts import timeout_to_seconds @@ -39,62 +47,27 @@ class RustAtranscription(Protocol): raise NotImplementedError -class _Unset: - pass - - -_UNSET: Final[_Unset] = _Unset() - - -@dataclass -class _RustTranscriptionState: - transcription: RustTranscription | None = None - atranscription: RustAtranscription | None = None - - -_STATE: Final = _RustTranscriptionState() +_TRANSCRIPTION: Final = NativeBinding[RustTranscription]("transcription") +_ATRANSCRIPTION: Final = NativeBinding[RustAtranscription]("atranscription") def configure_rust_transcription( enabled: bool = True, *, - transcription: RustTranscription | None | _Unset = _UNSET, - atranscription: RustAtranscription | None | _Unset = _UNSET, + transcription: RustTranscription | None | Unset = UNSET, + atranscription: RustAtranscription | None | Unset = UNSET, ) -> None: - if not isinstance(transcription, _Unset): - _STATE.transcription = transcription - if not isinstance(atranscription, _Unset): - _STATE.atranscription = atranscription + _ = enabled + _TRANSCRIPTION.update(transcription) + _ATRANSCRIPTION.update(atranscription) def load_rust_transcription() -> RustTranscription | None: - if _STATE.transcription is not None: - return _STATE.transcription - from litellm.rust_bridge import get_native_bridge - - native_bridge: Final = get_native_bridge() - return ( - None - if native_bridge is None - else cast( # cast-ok: native extension protocol is runtime-defined - RustTranscription, getattr(native_bridge, "transcription", None) - ) - ) + return _TRANSCRIPTION.load() def load_rust_atranscription() -> RustAtranscription | None: - if _STATE.atranscription is not None: - return _STATE.atranscription - from litellm.rust_bridge import get_native_bridge - - native_bridge: Final = get_native_bridge() - return ( - None - if native_bridge is None - else cast( # cast-ok: native extension protocol is runtime-defined - RustAtranscription, getattr(native_bridge, "atranscription", None) - ) - ) + return _ATRANSCRIPTION.load() def transcription( @@ -109,17 +82,26 @@ def transcription( timeout: float | httpx.Timeout | None, ) -> dict[str, object] | None: rust_transcription: Final = load_rust_transcription() - if rust_transcription is None: - return None - return rust_transcription( - model=model, - audio=audio, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - optional_params=optional_params, - timeout_seconds=timeout_to_seconds(timeout), + native_call: Final = ( + None + if rust_transcription is None + else lambda: rust_transcription( + model=model, + audio=audio, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + optional_params=optional_params, + timeout_seconds=timeout_to_seconds(timeout), + ) + ) + return invoke( + native_call=native_call, + fallback=lambda: None, + adapt=identity, + mode=FallbackMode.RUST_REQUIRED, + context=BridgeErrorContext(route="audio transcription", provider=custom_llm_provider or "", model=model), ) @@ -135,15 +117,24 @@ async def atranscription( timeout: float | httpx.Timeout | None, ) -> dict[str, object] | None: rust_atranscription: Final = load_rust_atranscription() - if rust_atranscription is None: - return None - return await rust_atranscription( - model=model, - audio=audio, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - optional_params=optional_params, - timeout_seconds=timeout_to_seconds(timeout), + native_call: Final = ( + None + if rust_atranscription is None + else lambda: rust_atranscription( + model=model, + audio=audio, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + optional_params=optional_params, + timeout_seconds=timeout_to_seconds(timeout), + ) + ) + return await ainvoke( + native_call=native_call, + fallback=async_none, + adapt=identity, + mode=FallbackMode.RUST_REQUIRED, + context=BridgeErrorContext(route="audio transcription", provider=custom_llm_provider or "", model=model), ) diff --git a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py index 90a5c8c35f3..362d3a5f5a0 100644 --- a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py +++ b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py @@ -10,7 +10,7 @@ import pytest import litellm from litellm.exceptions import APIError from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.rust_bridge import configuration +from litellm.rust_bridge import bindings, configuration from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -160,11 +160,7 @@ def test_load_rust_amessages_returns_injected_impl(): def test_messages_wrapper_returns_none_when_bridge_absent(monkeypatch): - monkeypatch.setattr( - importlib.import_module("litellm.rust_bridge"), - "get_native_bridge", - lambda: None, - ) + monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) litellm.use_litellm_rust(True) assert rust_messages.load_rust_messages() is None result = rust_messages.messages( @@ -448,11 +444,7 @@ async def test_fake_stream_wraps_rust_response_as_anthropic_sse(): @pytest.mark.asyncio async def test_gate_falls_back_when_bridge_unavailable(monkeypatch): - monkeypatch.setattr( - importlib.import_module("litellm.rust_bridge"), - "get_native_bridge", - lambda: None, - ) + monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) litellm.use_litellm_rust(True) response = await _gate() diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index 043537f8c1f..1d9a426283a 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -10,6 +10,7 @@ import litellm from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.llms.anthropic.chat.handler import ModelResponseIterator, make_call from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.rust_bridge import bindings as bridge_bindings from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, @@ -2318,14 +2319,13 @@ class TestRustChatCompletionsHook: bridge.set_rust_chat_completions(decline=gate, chat_completions=native) return seen - def test_rust_true_serves_the_call_and_stamps_the_header(self): + def test_rust_true_serves_the_call(self): from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion seen = self._inject() response = AnthropicChatCompletion().completion(**self._completion_kwargs()) assert response.choices[0].message.content == "hello from rust" - assert response._hidden_params["additional_headers"] == {"x-litellm-rust": "true"} assert len(seen["call"]) == 1 def test_the_core_receives_the_untranslated_openai_messages(self): @@ -2466,7 +2466,7 @@ class TestRustChatCompletionsHook: def declining_native(**_kwargs): raise _Declined("blank message text") - monkeypatch.setattr(bridge, "get_native_bridge", lambda: _FakeNative()) + monkeypatch.setattr(bridge_bindings, "get_native_bridge", lambda: _FakeNative()) bridge.set_rust_chat_completions( decline=lambda **_kwargs: None, chat_completions=declining_native ) @@ -2498,7 +2498,7 @@ class TestRustChatCompletionsHook: RustBridgeDeclined = _Declined RustUpstreamError = type("_Upstream", (Exception,), {}) - monkeypatch.setattr(bridge, "get_native_bridge", lambda: _FakeNative()) + monkeypatch.setattr(bridge_bindings, "get_native_bridge", lambda: _FakeNative()) async def declining_native(**_kwargs): raise _Declined("blank message text") @@ -2540,7 +2540,6 @@ class TestRustChatCompletionsHook: ) assert result.choices[0].message.content == "hello from rust" - assert result._hidden_params["additional_headers"] == {"x-litellm-rust": "true"} assert not python_call.called @@ -2558,7 +2557,7 @@ class TestRustChatCompletionsHook: RustBridgeDeclined = _Declined RustUpstreamError = type("_Upstream", (Exception,), {}) - monkeypatch.setattr(bridge, "get_native_bridge", lambda: _FakeNative()) + monkeypatch.setattr(bridge_bindings, "get_native_bridge", lambda: _FakeNative()) def declining_native(**_kwargs): raise _Declined("blank message text") diff --git a/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py index 21e3239f623..8ac88a8172b 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -10,10 +10,11 @@ from unittest.mock import MagicMock, patch import httpx import pytest - from botocore.credentials import Credentials + from litellm.llms.bedrock.chat.converse_handler import BedrockConverseLLM from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.rust_bridge import bindings as bridge_bindings from litellm.rust_bridge import chat_completions as bridge from litellm.types.utils import ModelResponse @@ -111,12 +112,11 @@ def _recording_logging_obj(): return logging_obj, calls -def test_rust_true_serves_the_call_and_stamps_the_header(): +def test_rust_true_serves_the_call(): seen = _inject() response = _run() assert response.choices[0].message.content == "hello from rust" - assert response._hidden_params["additional_headers"] == {"x-litellm-rust": "true"} assert len(seen["call"]) == 1 @@ -204,7 +204,7 @@ async def test_the_async_path_falls_back_when_the_core_declines(monkeypatch): RustBridgeDeclined = _Declined RustUpstreamError = type("_Upstream", (Exception,), {}) - monkeypatch.setattr(bridge, "get_native_bridge", lambda: _FakeNative()) + monkeypatch.setattr(bridge_bindings, "get_native_bridge", lambda: _FakeNative()) async def declining_native(**_kwargs): raise _Declined("blank message text") @@ -254,7 +254,6 @@ async def test_the_async_path_serves_the_rust_response_without_the_fallback(): ) assert result.choices[0].message.content == "hello from rust" - assert result._hidden_params["additional_headers"] == {"x-litellm-rust": "true"} assert not python_call.called @@ -281,7 +280,7 @@ async def test_pre_call_logging_fires_once_even_when_the_rust_path_declines(): return ModelResponse() with ( - patch.object(bridge, "get_native_bridge", lambda: _FakeNative()), + patch.object(bridge_bindings, "get_native_bridge", lambda: _FakeNative()), patch.object( BedrockConverseLLM, "get_credentials", return_value=RESOLVED_CREDENTIALS ), @@ -388,7 +387,7 @@ def test_pre_call_logging_fires_once_when_the_sync_rust_path_declines(): logging_obj = MagicMock() - with patch.object(bridge, "get_native_bridge", lambda: _FakeNative()): + with patch.object(bridge_bindings, "get_native_bridge", lambda: _FakeNative()): bridge.set_rust_chat_completions( decline=lambda **_kwargs: None, chat_completions=declining_native ) @@ -473,7 +472,7 @@ def test_post_call_is_not_logged_twice_when_the_sync_rust_call_declines(): logging_obj, calls = _recording_logging_obj() - with patch.object(bridge, "get_native_bridge", lambda: _FakeNative()): + with patch.object(bridge_bindings, "get_native_bridge", lambda: _FakeNative()): bridge.set_rust_chat_completions( decline=lambda **_kwargs: None, chat_completions=declining_native ) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index 0764aec7185..e8f3ea3a0a9 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -10,7 +10,9 @@ import pytest import litellm from litellm.llms.base_llm.ocr.transformation import OCRResponse -from litellm.rust_bridge import configuration +from litellm.rust_bridge import bindings, configuration +from litellm.rust_bridge.bindings import UNSET, Unset +from litellm.rust_bridge.ocr import RustAocr, RustOcr # `litellm/__init__.py` does `from .ocr.main import *`, which binds the `ocr` # function onto `litellm.ocr` and shadows the submodule, so import the modules @@ -34,6 +36,16 @@ FAKE_OCR_RESPONSE: dict[str, object] = { } +def _use_test_rust( + enabled: bool = True, + *, + ocr: RustOcr | None | Unset = UNSET, + aocr: RustAocr | None | Unset = UNSET, +) -> None: + rust_bridge.set_rust_ocr(ocr=ocr, aocr=aocr) + litellm.use_litellm_rust(enabled) + + class CapturedException(Exception): pass @@ -228,7 +240,8 @@ def _reset_rust_flag(): def fake_bridge(): """Enable the Rust path with an injected recording bridge (no native wheel).""" bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + rust_bridge.set_rust_ocr(ocr=bridge) + litellm.use_litellm_rust(True) return bridge @@ -236,7 +249,8 @@ def fake_bridge(): def fake_async_bridge(): """Enable the async Rust path with an injected recording bridge.""" bridge = RecordingAsyncBridge() - litellm.use_litellm_rust(True, aocr=bridge) + rust_bridge.set_rust_ocr(aocr=bridge) + litellm.use_litellm_rust(True) return bridge @@ -248,21 +262,16 @@ def test_use_litellm_rust_toggles_flag(): assert rust_bridge.rust_ocr_enabled() is False -def test_env_var_enables_rust_ocr(monkeypatch): - monkeypatch.setenv("LITELLM_USE_RUST_OCR", "1") - with pytest.warns(DeprecationWarning, match="LITELLM_USE_RUST_OCR is deprecated"): - assert rust_bridge.rust_ocr_enabled() is True - - -def test_explicit_false_overrides_process_enable(): +def test_explicit_false_overrides_the_process_switch(): litellm.use_litellm_rust(True) + prepared = build_prepared_request(litellm_params={"rust": False}) - assert ocr_main._rust_ocr_enabled(build_prepared_request(litellm_params={"rust": False})) is False + assert ocr_main._rust_ocr_enabled(prepared) is False def test_load_rust_ocr_returns_injected_impl(): bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + _use_test_rust(True, ocr=bridge) assert rust_bridge.load_rust_ocr() is bridge @@ -306,40 +315,21 @@ def test_native_bridge_available_reflects_loader(monkeypatch): def test_load_rust_aocr_returns_injected_impl(): bridge = RecordingAsyncBridge() - litellm.use_litellm_rust(True, aocr=bridge) + _use_test_rust(True, aocr=bridge) assert rust_bridge.load_rust_aocr() is bridge -def test_toggle_without_ocr_arg_preserves_injected_impl(): - """Regression: routine enable/disable calls must not clobber a prior injection. - - Earlier, ``use_litellm_rust()`` unconditionally assigned the keyword default - of ``None`` to ``_rust_ocr_impl``, silently dropping a custom bridge whenever - a caller toggled the flag without re-passing ``ocr=``. - """ - bridge = RecordingBridge() - async_bridge = RecordingAsyncBridge() - litellm.use_litellm_rust(True, ocr=bridge, aocr=async_bridge) - - litellm.use_litellm_rust(False) - assert rust_bridge.load_rust_ocr() is bridge - assert rust_bridge.load_rust_aocr() is async_bridge - litellm.use_litellm_rust(True) - assert rust_bridge.load_rust_ocr() is bridge - assert rust_bridge.load_rust_aocr() is async_bridge - - def test_explicit_ocr_none_clears_injected_impl(monkeypatch): monkeypatch.setattr( - importlib.import_module("litellm.rust_bridge"), + bindings, "get_native_bridge", lambda: None, ) bridge = RecordingBridge() async_bridge = RecordingAsyncBridge() - litellm.use_litellm_rust(True, ocr=bridge, aocr=async_bridge) + rust_bridge.set_rust_ocr(ocr=bridge, aocr=async_bridge) - litellm.use_litellm_rust(True, ocr=None, aocr=None) + rust_bridge.set_rust_ocr(ocr=None, aocr=None) assert rust_bridge.load_rust_ocr() is None assert rust_bridge.load_rust_aocr() is None @@ -348,7 +338,7 @@ def test_load_rust_ocr_none_when_extension_absent(monkeypatch): """With no injected impl and no compiled wheel, the loader returns None so the caller degrades to the Python path instead of raising ImportError.""" monkeypatch.setattr( - importlib.import_module("litellm.rust_bridge"), + bindings, "get_native_bridge", lambda: None, ) @@ -365,7 +355,7 @@ def test_load_rust_ocr_uses_compiled_extension(monkeypatch): fake_module.ocr = lambda **kwargs: dict(FAKE_OCR_RESPONSE) # type: ignore[attr-defined] fake_module.aocr = lambda **kwargs: dict(FAKE_OCR_RESPONSE) # type: ignore[attr-defined] monkeypatch.setattr( - importlib.import_module("litellm.rust_bridge"), + bindings, "get_native_bridge", lambda: fake_module, ) @@ -384,7 +374,7 @@ def test_timeout_to_seconds_handles_float_timeout_and_none(): def test_bridge_wrapper_forwards_prepared_args_and_wraps_response(): bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + _use_test_rust(True, ocr=bridge) response = rust_bridge.ocr( model="mistral-ocr-latest", document=DOCUMENT, @@ -417,7 +407,7 @@ def test_bridge_wrapper_forwards_prepared_args_and_wraps_response(): async def test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response(): bridge = RecordingAsyncBridge() - litellm.use_litellm_rust(True, aocr=bridge) + _use_test_rust(True, aocr=bridge) response = await rust_bridge.aocr( model="mistral-ocr-maas", document=DOCUMENT, @@ -445,7 +435,7 @@ async def test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response(): def test_run_rust_ocr_prepares_request_and_wraps_response(): bridge = RecordingBridge() logging_obj = RecordingLogging() - litellm.use_litellm_rust(True, ocr=bridge) + _use_test_rust(True, ocr=bridge) response = ocr_main._run_rust_ocr( prepared_request=build_prepared_request( @@ -477,7 +467,7 @@ def test_run_rust_ocr_prepares_request_and_wraps_response(): def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing(): bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + _use_test_rust(True, ocr=bridge) ocr_main._run_rust_ocr( prepared_request=build_prepared_request(api_key=None, timeout=None), @@ -489,7 +479,7 @@ def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing(): def test_run_rust_ocr_prefers_explicit_key_over_resolver(): bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + _use_test_rust(True, ocr=bridge) def _resolver(name: str) -> str | None: raise AssertionError(f"resolver should not be called for {name}") @@ -508,7 +498,7 @@ def test_run_rust_ocr_prefers_explicit_key_over_resolver(): def test_run_rust_ocr_uses_provider_api_key_env_var(): bridge = RecordingBridge() resolver_calls = [] - litellm.use_litellm_rust(True, ocr=bridge) + _use_test_rust(True, ocr=bridge) def _resolver(name): resolver_calls.append(name) @@ -530,7 +520,7 @@ def test_run_rust_ocr_uses_provider_api_key_env_var(): def test_prepare_rust_ocr_call_forwards_vertex_routing_metadata(): bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + _use_test_rust(True, ocr=bridge) ocr_main._run_rust_ocr( prepared_request=build_prepared_request( @@ -556,7 +546,7 @@ def test_prepare_rust_ocr_call_forwards_vertex_routing_metadata(): def test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_manager(): bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + _use_test_rust(True, ocr=bridge) def _resolver(name: str) -> str | None: return { @@ -579,7 +569,7 @@ def test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_mana def test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager(): bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + _use_test_rust(True, ocr=bridge) ocr_main._run_rust_ocr( prepared_request=build_prepared_request( @@ -596,7 +586,7 @@ def test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager(): def test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint(): bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + _use_test_rust(True, ocr=bridge) ocr_main._run_rust_ocr( prepared_request=build_prepared_request( @@ -616,7 +606,7 @@ def test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint(): def test_run_rust_ocr_runs_pre_call_logging(): logging_obj = RecordingLogging() bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + _use_test_rust(True, ocr=bridge) ocr_main._run_rust_ocr( prepared_request=build_prepared_request( @@ -703,7 +693,7 @@ def test_ocr_exception_type_uses_resolved_provider_context( return CapturedException("wrapped") monkeypatch.setattr(ocr_main.litellm, "exception_type", fake_exception_type) - litellm.use_litellm_rust(True, ocr=RaisingBridge()) + _use_test_rust(True, ocr=RaisingBridge()) with pytest.raises(CapturedException): litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") @@ -748,7 +738,7 @@ async def test_aocr_exception_type_uses_resolved_provider_context( return CapturedException("wrapped") monkeypatch.setattr(ocr_main.litellm, "exception_type", fake_exception_type) - litellm.use_litellm_rust(True, aocr=RaisingAsyncBridge()) + _use_test_rust(True, aocr=RaisingAsyncBridge()) with pytest.raises(CapturedException): await litellm.aocr(model=MODEL, document=DOCUMENT, api_key="sk-test") @@ -776,7 +766,7 @@ def test_ocr_passes_default_request_timeout_to_rust(fake_bridge): def test_ocr_does_not_route_to_rust_when_disabled(): """With the flag off, the bridge must not be consulted even if an impl exists.""" bridge = RecordingBridge() - litellm.use_litellm_rust(False, ocr=bridge) + _use_test_rust(False, ocr=bridge) assert rust_bridge.rust_ocr_enabled() is False # The impl stays available for injection, but the disabled flag gates usage, @@ -804,6 +794,25 @@ def test_ocr_falls_back_to_python_when_bridge_unavailable(monkeypatch): assert isinstance(response, OCRResponse) +def test_ocr_unsupported_provider_skips_rust(monkeypatch): + bridge = RecordingBridge() + _use_test_rust(True, ocr=bridge) + + def fake_handler_ocr(**kwargs): + return OCRResponse(pages=[], model="parse-v3", object="ocr") + + monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", fake_handler_ocr) + + response = litellm.ocr( + model="reducto/parse-v3", + document={"type": "document_url", "document_url": "reducto://document-id"}, + api_key="test-key", + ) + + assert isinstance(response, OCRResponse) + assert bridge.calls == [] + + def test_ocr_provider_configs_expose_api_key_env_vars(): from litellm.llms.azure_ai.ocr.document_intelligence.transformation import ( AzureDocumentIntelligenceOCRConfig, diff --git a/tests/test_litellm/responses/test_rust_bridge_websocket.py b/tests/test_litellm/responses/test_rust_bridge_websocket.py index 1233ddf1785..9aa9c4acd3d 100644 --- a/tests/test_litellm/responses/test_rust_bridge_websocket.py +++ b/tests/test_litellm/responses/test_rust_bridge_websocket.py @@ -3,7 +3,7 @@ from __future__ import annotations import pytest from litellm.llms.custom_httpx.llm_http_handler import _rust_responses_websocket_enabled -from litellm.rust_bridge import configuration, responses_websocket +from litellm.rust_bridge import bindings, configuration, responses_websocket from litellm.types.router import GenericLiteLLMParams @@ -76,8 +76,7 @@ async def test_adapter_raises_clean_close_when_rust_connection_ends() -> None: @pytest.mark.asyncio async def test_bridge_unavailable_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(responses_websocket, "_STATE", responses_websocket._RustResponsesWebSocketState()) - monkeypatch.setattr(responses_websocket, "get_native_bridge", lambda: None) + monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) assert ( await responses_websocket.connect( diff --git a/tests/test_litellm/rust_bridge/test_chat_completions.py b/tests/test_litellm/rust_bridge/test_chat_completions.py index 03921133c77..a7c8570e78d 100644 --- a/tests/test_litellm/rust_bridge/test_chat_completions.py +++ b/tests/test_litellm/rust_bridge/test_chat_completions.py @@ -10,8 +10,9 @@ from __future__ import annotations import pytest import litellm +from litellm.exceptions import APIError +from litellm.rust_bridge import bindings, configuration from litellm.rust_bridge import chat_completions as bridge -from litellm.rust_bridge import configuration from litellm.types.utils import ModelResponse RUST_RESPONSE = { @@ -54,7 +55,7 @@ class _FakeNative: def _fake_native_bridge(monkeypatch): """Expose the bridge's exception classes without the compiled extension.""" - monkeypatch.setattr(bridge, "get_native_bridge", lambda: _FakeNative()) + monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) def _hide_native_bridge(monkeypatch): @@ -63,31 +64,23 @@ def _hide_native_bridge(monkeypatch): There is no injection seam for "the .so is absent", so the loader itself is replaced; every other case here uses `set_rust_chat_completions`. """ - monkeypatch.setattr(bridge, "get_native_bridge", lambda: None) + monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) @pytest.fixture(autouse=True) def reset_bridge(): """Every test starts with no injected callables, and leaves none behind.""" - bridge.set_rust_chat_completions(chat_completions=None, achat_completions=None, decline=None) + bridge.set_rust_chat_completions( + chat_completions=None, + achat_completions=None, + decline=lambda **_kwargs: None, + ) configuration.reset_rust_configuration() yield bridge.set_rust_chat_completions(chat_completions=None, achat_completions=None, decline=None) configuration.reset_rust_configuration() -class _RecordingDecline: - """A stand-in for the native gate that records what it was asked.""" - - def __init__(self, reason: str | None = None): - self.reason = reason - self.calls: list[dict] = [] - - def __call__(self, **kwargs): - self.calls.append(kwargs) - return self.reason - - class _RecordingCall: def __init__(self, result=None, error: Exception | None = None): self.result = result if result is not None else dict(RUST_RESPONSE) @@ -106,7 +99,7 @@ class _RecordingAsyncCall(_RecordingCall): return _RecordingCall.__call__(self, **kwargs) -def _accepts(**overrides) -> bool: +def _should_attempt(**overrides) -> bool: kwargs = { "model": "claude-sonnet-4-5", "messages": MESSAGES, @@ -116,52 +109,40 @@ def _accepts(**overrides) -> bool: "stream": None, } kwargs.update(overrides) + kwargs.pop("asynchronous", None) return bridge.rust_chat_completions_accepts(**kwargs) -class TestGate: +class TestEligibility: def test_declines_when_the_deployment_did_not_opt_in(self, monkeypatch): monkeypatch.delenv("LITELLM_RUST", raising=False) - gate = _RecordingDecline() - bridge.set_rust_chat_completions(decline=gate) - assert _accepts(litellm_params={}) is False - assert _accepts(litellm_params=None) is False - assert _accepts(litellm_params={"rust": False}) is False - assert gate.calls == [], "the gate must not be consulted before opt-in" + bridge.set_rust_chat_completions(chat_completions=_RecordingCall()) + assert _should_attempt(litellm_params={}) is False + assert _should_attempt(litellm_params=None) is False + assert _should_attempt(litellm_params={"rust": False}) is False - def test_accepts_when_the_deployment_opted_in_and_the_core_agrees(self, monkeypatch): + def test_attempts_when_the_deployment_opted_in(self, monkeypatch): monkeypatch.delenv("LITELLM_RUST", raising=False) - gate = _RecordingDecline() - bridge.set_rust_chat_completions(decline=gate) - assert _accepts() is True - assert gate.calls[0]["model"] == "claude-sonnet-4-5" - assert gate.calls[0]["custom_llm_provider"] == "anthropic" - - def test_explicit_false_overrides_process_enable(self): - bridge.set_rust_chat_completions(decline=_RecordingDecline()) - configuration.use_litellm_rust(True) - - assert _accepts(litellm_params={"rust": False}) is False - - def test_process_enable_applies_without_request_override(self): - bridge.set_rust_chat_completions(decline=_RecordingDecline()) - configuration.use_litellm_rust(True) - - assert _accepts(litellm_params={}) is True + bridge.set_rust_chat_completions(chat_completions=_RecordingCall()) + assert _should_attempt() is True def test_the_env_var_opts_in_without_a_per_model_flag(self, monkeypatch): monkeypatch.setenv("LITELLM_RUST", "true") - bridge.set_rust_chat_completions(decline=_RecordingDecline()) - assert _accepts(litellm_params={}) is True + bridge.set_rust_chat_completions(chat_completions=_RecordingCall()) + assert _should_attempt(litellm_params={}) is True + + def test_explicit_false_overrides_the_process_switch(self): + bridge.set_rust_chat_completions(chat_completions=_RecordingCall()) + configuration.use_litellm_rust(True) + + assert _should_attempt(litellm_params={"rust": False}) is False def test_declines_streaming_and_providers_off_the_path(self, monkeypatch): monkeypatch.delenv("LITELLM_RUST", raising=False) - gate = _RecordingDecline() - bridge.set_rust_chat_completions(decline=gate) - assert _accepts(stream=True) is False - assert _accepts(custom_llm_provider="openai") is False - assert _accepts(custom_llm_provider=None) is False - assert gate.calls == [] + bridge.set_rust_chat_completions(chat_completions=_RecordingCall()) + assert _should_attempt(stream=True) is False + assert _should_attempt(custom_llm_provider="openai") is False + assert _should_attempt(custom_llm_provider=None) is False def test_declines_an_anthropic_request_carrying_a_litellm_metadata_user_id(self, monkeypatch): """`AnthropicConfig.transform_request` copies a valid `user_id` into the Messages body. @@ -171,24 +152,21 @@ class TestGate: to Anthropic with the abuse-detection attribution silently missing. """ monkeypatch.delenv("LITELLM_RUST", raising=False) - gate = _RecordingDecline() - bridge.set_rust_chat_completions(decline=gate) - assert _accepts(litellm_params={"rust": True, "metadata": {"user_id": "u-123"}}) is False - assert gate.calls == [], "the core must not be consulted for a request it cannot see the key of" + bridge.set_rust_chat_completions(chat_completions=_RecordingCall()) + assert _should_attempt(litellm_params={"rust": True, "metadata": {"user_id": "u-123"}}) is False # Bedrock's Converse transform reads no `user_id`, and an Anthropic request # whose metadata carries none is one Python would not attribute either. assert ( - _accepts( + _should_attempt( custom_llm_provider="bedrock", - model="bedrock/us-east-1/anthropic.claude-v2", litellm_params={"rust": True, "metadata": {"user_id": "u-123"}}, ) is True ) - assert _accepts(litellm_params={"rust": True, "metadata": {"trace_id": "t-1"}}) is True - assert _accepts(litellm_params={"rust": True, "metadata": {"user_id": None}}) is True - assert _accepts(litellm_params={"rust": True, "metadata": None}) is True + assert _should_attempt(litellm_params={"rust": True, "metadata": {"trace_id": "t-1"}}) is True + assert _should_attempt(litellm_params={"rust": True, "metadata": {"user_id": None}}) is True + assert _should_attempt(litellm_params={"rust": True, "metadata": None}) is True def test_declines_a_bedrock_request_while_the_proxy_owns_request_metadata(self, monkeypatch): """`AmazonConverseConfig` resolves proxy-owned `requestMetadata` onto the @@ -197,39 +175,49 @@ class TestGate: who armed `bedrock_request_metadata_fields` keeps the Python path. """ monkeypatch.delenv("LITELLM_RUST", raising=False) - gate = _RecordingDecline() - bridge.set_rust_chat_completions(decline=gate) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall()) bedrock = { "custom_llm_provider": "bedrock", - "model": "bedrock/us-east-1/anthropic.claude-v2", } monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ["user_api_key_team_id"]) - assert _accepts(**bedrock) is False - assert gate.calls == [], "the core must not be consulted for a field it cannot write" - assert _accepts() is True, "arming Bedrock attribution must not decline Anthropic" + assert _should_attempt(**bedrock) is False + assert _should_attempt() is True, "arming Bedrock attribution must not decline Anthropic" monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", None) - assert _accepts(**bedrock) is True, "the decline follows the operator's opt-in alone" - - def test_declines_when_the_core_declines(self, monkeypatch): - monkeypatch.delenv("LITELLM_RUST", raising=False) - bridge.set_rust_chat_completions(decline=_RecordingDecline("streaming")) - assert _accepts() is False + assert _should_attempt(**bedrock) is True, "the decline follows the operator's opt-in alone" def test_declines_when_the_bridge_is_unavailable(self, monkeypatch): monkeypatch.delenv("LITELLM_RUST", raising=False) + bridge.set_rust_chat_completions(decline=None) _hide_native_bridge(monkeypatch) - assert _accepts() is False + assert _should_attempt() is False - def test_declines_when_the_gate_itself_raises(self, monkeypatch): - monkeypatch.delenv("LITELLM_RUST", raising=False) + def test_checks_the_native_capability_gate(self, monkeypatch): + _hide_native_bridge(monkeypatch) + bridge.set_rust_chat_completions(decline=None) + assert _should_attempt() is False + bridge.set_rust_chat_completions(decline=lambda **_kwargs: None) + assert _should_attempt() is True - def exploding(**_kwargs): - raise RuntimeError("boom") + def test_native_unsupported_capability_declines(self, monkeypatch): + _fake_native_bridge(monkeypatch) - bridge.set_rust_chat_completions(decline=exploding) - assert _accepts() is False + def decline(**_kwargs): + raise _FakeDeclined("unsupported request") + + bridge.set_rust_chat_completions(decline=decline) + assert _should_attempt() is False + + def test_native_capability_failure_does_not_fall_back(self, monkeypatch): + _fake_native_bridge(monkeypatch) + + def fail(**_kwargs): + raise _FakeUpstream(502, "capability check failed") + + bridge.set_rust_chat_completions(decline=fail) + with pytest.raises(APIError, match="capability check failed"): + _should_attempt() def _call_kwargs(model_response: ModelResponse) -> dict: @@ -247,14 +235,26 @@ def _call_kwargs(model_response: ModelResponse) -> dict: } +def _sync_call_kwargs(model_response: ModelResponse) -> dict: + return {**_call_kwargs(model_response), "python_fallback": lambda: "python"} + + +async def _async_python_fallback() -> object: + return "python" + + +def _async_call_kwargs(model_response: ModelResponse) -> dict: + return {**_call_kwargs(model_response), "python_fallback": _async_python_fallback} + + class TestSyncCall: - def test_builds_a_model_response_and_stamps_the_rust_header(self): + def test_builds_a_model_response(self): native = _RecordingCall() bridge.set_rust_chat_completions(chat_completions=native) model_response = ModelResponse() original_id = model_response.id - result = bridge.chat_completions(**_call_kwargs(model_response)) + result = bridge.chat_completions_or_fallback(**_sync_call_kwargs(model_response)) assert result is not None assert result.choices[0].message.content == "hello from rust" @@ -263,44 +263,54 @@ class TestSyncCall: assert result.usage.prompt_tokens == 11 assert result.usage.completion_tokens == 4 assert result.usage.total_tokens == 15 - assert result._hidden_params["additional_headers"] == {"x-litellm-rust": "true"} assert result.id == original_id, "the rust path must keep the chatcmpl id litellm already minted" def test_passes_the_timeout_through_as_seconds(self): native = _RecordingCall() bridge.set_rust_chat_completions(chat_completions=native) - bridge.chat_completions(**_call_kwargs(ModelResponse())) + bridge.chat_completions_or_fallback(**_sync_call_kwargs(ModelResponse())) assert native.calls[0]["timeout_seconds"] == 30.0 def test_falls_back_when_the_bridge_is_unavailable(self, monkeypatch): _hide_native_bridge(monkeypatch) - assert bridge.chat_completions(**_call_kwargs(ModelResponse())) is None + assert bridge.chat_completions_or_fallback(**_sync_call_kwargs(ModelResponse())) == "python" def test_falls_back_when_the_core_declines_before_calling_the_provider(self, monkeypatch): _fake_native_bridge(monkeypatch) bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeDeclined("streaming"))) - assert bridge.chat_completions(**_call_kwargs(ModelResponse())) is None + assert bridge.chat_completions_or_fallback(**_sync_call_kwargs(ModelResponse())) == "python" + + def test_model_response_fallback_is_returned_unchanged(self, monkeypatch): + _fake_native_bridge(monkeypatch) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeDeclined("unsupported"))) + fallback_response = ModelResponse() + + result = bridge.chat_completions_or_fallback( + **_call_kwargs(ModelResponse()), + python_fallback=lambda: fallback_response, + ) + + assert result is fallback_response class TestAsyncCall: @pytest.mark.asyncio async def test_builds_a_model_response(self): bridge.set_rust_chat_completions(achat_completions=_RecordingAsyncCall()) - result = await bridge.achat_completions(**_call_kwargs(ModelResponse())) + result = await bridge.achat_completions_or_fallback(**_async_call_kwargs(ModelResponse())) assert result is not None assert result.choices[0].message.content == "hello from rust" - assert result._hidden_params["additional_headers"] == {"x-litellm-rust": "true"} @pytest.mark.asyncio async def test_falls_back_when_the_bridge_is_unavailable(self, monkeypatch): _hide_native_bridge(monkeypatch) - assert await bridge.achat_completions(**_call_kwargs(ModelResponse())) is None + assert await bridge.achat_completions_or_fallback(**_async_call_kwargs(ModelResponse())) == "python" @pytest.mark.asyncio async def test_falls_back_when_the_core_declines_before_calling_the_provider(self, monkeypatch): _fake_native_bridge(monkeypatch) bridge.set_rust_chat_completions(achat_completions=_RecordingAsyncCall(error=_FakeDeclined("streaming"))) - assert await bridge.achat_completions(**_call_kwargs(ModelResponse())) is None + assert await bridge.achat_completions_or_fallback(**_async_call_kwargs(ModelResponse())) == "python" class TestAsyncFallbackWrapper: @@ -349,14 +359,14 @@ class TestFailureClassification: def test_a_decline_falls_back_because_nothing_was_sent(self): bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeDeclined("streaming"))) - assert bridge.chat_completions(**_call_kwargs(ModelResponse())) is None + assert bridge.chat_completions_or_fallback(**_sync_call_kwargs(ModelResponse())) == "python" def test_an_upstream_failure_is_surfaced_with_its_status(self): from litellm.exceptions import APIError bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeUpstream(429, "429: rate limited"))) with pytest.raises(APIError) as raised: - bridge.chat_completions(**_call_kwargs(ModelResponse())) + bridge.chat_completions_or_fallback(**_sync_call_kwargs(ModelResponse())) assert raised.value.status_code == 429 assert "rate limited" in str(raised.value) @@ -365,13 +375,13 @@ class TestFailureClassification: bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeUpstream(0, "connection reset"))) with pytest.raises(APIError) as raised: - bridge.chat_completions(**_call_kwargs(ModelResponse())) + bridge.chat_completions_or_fallback(**_sync_call_kwargs(ModelResponse())) assert raised.value.status_code == 500 def test_an_unrecognized_error_is_not_swallowed(self): bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=RuntimeError("something else"))) with pytest.raises(RuntimeError): - bridge.chat_completions(**_call_kwargs(ModelResponse())) + bridge.chat_completions_or_fallback(**_sync_call_kwargs(ModelResponse())) @pytest.mark.asyncio async def test_the_async_wrapper_does_not_fall_back_on_an_upstream_failure(self): diff --git a/tests/test_litellm/test_audio_transcription_rust_bridge.py b/tests/test_litellm/test_audio_transcription_rust_bridge.py index bbeb6c38f78..92e86f48c31 100644 --- a/tests/test_litellm/test_audio_transcription_rust_bridge.py +++ b/tests/test_litellm/test_audio_transcription_rust_bridge.py @@ -6,6 +6,7 @@ import litellm from litellm.llms.bedrock.audio_transcription import BedrockAudioTranscriptionRustDispatch rust_bridge = importlib.import_module("litellm.rust_bridge.transcription") +rust_bridge_bindings = importlib.import_module("litellm.rust_bridge.bindings") class SyncBridge: @@ -77,7 +78,7 @@ async def test_enabled_async_bridge() -> None: def test_loader_returns_none_without_native_extension(monkeypatch: pytest.MonkeyPatch) -> None: rust_bridge.configure_rust_transcription(transcription=None, atranscription=None) - monkeypatch.setattr("litellm.rust_bridge.get_native_bridge", lambda: None) + monkeypatch.setattr(rust_bridge_bindings, "get_native_bridge", lambda: None) assert rust_bridge.load_rust_transcription() is None assert rust_bridge.load_rust_atranscription() is None From 9427b6e7085214be2d4a718bb56c1c5a85a834b9 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 06:56:42 -0700 Subject: [PATCH 60/67] feat(proxy): report Rust execution provenance --- .../ai-gateway/src/routes/messages/mod.rs | 21 ++++- .../messages/streaming_iterator.py | 10 ++- .../bedrock/audio_transcription/__init__.py | 9 ++- litellm/llms/custom_httpx/llm_http_handler.py | 39 +++++++-- litellm/ocr/main.py | 29 ++++++- litellm/proxy/common_request_processing.py | 14 +++- litellm/rust_bridge/chat_completions.py | 53 ++++++++++--- litellm/rust_bridge/messages.py | 42 +++++----- litellm/rust_bridge/ocr.py | 18 +++-- litellm/rust_bridge/responses_websocket.py | 9 ++- litellm/rust_bridge/runtime.py | 79 ++++++++++++++++--- litellm/rust_bridge/transcription.py | 18 +++-- .../test_rust_bridge_messages.py | 6 +- tests/test_litellm/ocr/test_rust_bridge.py | 9 +++ .../responses/test_rust_bridge_websocket.py | 18 +++-- .../rust_bridge/test_chat_completions.py | 8 ++ .../test_litellm/rust_bridge/test_runtime.py | 47 ++++++++--- .../test_audio_transcription_rust_bridge.py | 10 +++ 18 files changed, 341 insertions(+), 98 deletions(-) diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs index e4861c3954c..05c74a05924 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs @@ -4,9 +4,10 @@ mod service; use axum::Router; use axum::body::Body; -use axum::extract::{Json, State}; +use axum::extract::{Json, Request, State}; use axum::http::StatusCode; use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE, HeaderMap, HeaderValue}; +use axum::middleware::{self, Next}; use axum::response::{IntoResponse, Response}; use axum::routing::post; use litellm_core::Error; @@ -16,9 +17,23 @@ use crate::auth::RequireMasterKey; use crate::constants::{MESSAGES_HEADERS_NOT_FORWARDED, MESSAGES_ROUTE_PATH}; use crate::state::AppState; +const CORE_ENGINE_HEADER: &str = "x-litellm-core"; +const RUST_CORE_ENGINE: &str = "rust"; + /// This route's contribution to the app router. pub fn router() -> Router { - Router::new().route(MESSAGES_ROUTE_PATH, post(handle)) + Router::new() + .route(MESSAGES_ROUTE_PATH, post(handle)) + .route_layer(middleware::from_fn(core_engine_header)) +} + +async fn core_engine_header(request: Request, next: Next) -> Response { + let mut response = next.run(request).await; + response.headers_mut().insert( + CORE_ENGINE_HEADER, + HeaderValue::from_static(RUST_CORE_ENGINE), + ); + response } async fn handle( @@ -142,6 +157,7 @@ mod tests { use tower::ServiceExt; use super::super::app; + use super::{CORE_ENGINE_HEADER, RUST_CORE_ENGINE}; use crate::io::realtime_pool::RealtimePool; use crate::state::AppState; @@ -289,6 +305,7 @@ mod tests { .await .expect("route responds"); assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[CORE_ENGINE_HEADER], RUST_CORE_ENGINE); let body = axum::body::to_bytes(response.into_body(), usize::MAX) .await .expect("response body reads"); diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py index 66e36dab2ba..4424ef4a081 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py @@ -6,7 +6,7 @@ from typing import Any, Final, Protocol, runtime_checkable import httpx from pydantic import TypeAdapter -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.constants import ( ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS, @@ -19,6 +19,7 @@ from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) +from litellm.rust_bridge.runtime import CoreEngine, execution_additional_headers from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from litellm.types.utils import GenericStreamingChunk, ModelResponseStream @@ -325,7 +326,8 @@ def _anthropic_content_block_events(index: int, block: Mapping[str, object]) -> class AnthropicMessagesStreamHiddenParams(TypedDict): - additional_headers: dict[str, str] + additional_headers: ReadOnly[Mapping[str, str]] + core_engine: ReadOnly[str] @runtime_checkable @@ -344,8 +346,10 @@ _RESPONSE_HEADERS_ADAPTER: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict def anthropic_messages_stream_hidden_params( response_headers: httpx.Headers, ) -> AnthropicMessagesStreamHiddenParams: + additional_headers: Final = _RESPONSE_HEADERS_ADAPTER.validate_python(process_response_headers(response_headers)) return AnthropicMessagesStreamHiddenParams( - additional_headers=_RESPONSE_HEADERS_ADAPTER.validate_python(process_response_headers(response_headers)) + additional_headers=execution_additional_headers(additional_headers, CoreEngine.PYTHON), + core_engine=CoreEngine.PYTHON.value, ) diff --git a/litellm/llms/bedrock/audio_transcription/__init__.py b/litellm/llms/bedrock/audio_transcription/__init__.py index b1f8c957ff4..e1504a18f71 100644 --- a/litellm/llms/bedrock/audio_transcription/__init__.py +++ b/litellm/llms/bedrock/audio_transcription/__init__.py @@ -5,6 +5,7 @@ import httpx from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.rust_bridge import transcription as rust_transcription_bridge +from litellm.rust_bridge.runtime import CoreEngine, execution_hidden_params from litellm.types.utils import FileTypes, TranscriptionResponse @@ -55,7 +56,9 @@ class BedrockAudioTranscriptionRustDispatch: ) if rust_response is None: raise RuntimeError("Rust audio transcription bridge is unavailable") - return TranscriptionResponse(**rust_response) + response: Final = TranscriptionResponse(**rust_response) + response["_hidden_params"] = execution_hidden_params(None, CoreEngine.RUST) + return response async def async_audio_transcriptions( self, @@ -81,4 +84,6 @@ class BedrockAudioTranscriptionRustDispatch: ) if rust_response is None: raise RuntimeError("Rust audio transcription bridge is unavailable") - return TranscriptionResponse(**rust_response) + response: Final = TranscriptionResponse(**rust_response) + response["_hidden_params"] = execution_hidden_params(None, CoreEngine.RUST) + return response diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9aaf92f62e4..a00e0fa0ae1 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -19,7 +19,6 @@ import litellm.types.utils from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES -from litellm.exceptions import APIError from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, @@ -89,6 +88,7 @@ from litellm.responses.streaming_iterator import ( ResponsesWebSocketStreaming, SyncResponsesAPIStreamingIterator, ) +from litellm.rust_bridge.runtime import CoreEngine, execution_additional_headers, execution_hidden_params from litellm.types.containers.main import ( ContainerFileListResponse, ContainerListResponse, @@ -166,6 +166,18 @@ def _rust_responses_websocket_enabled( return custom_llm_provider == "openai" and rust_enabled(request_override=request_override) +def _anthropic_messages_with_core_engine( + response: AnthropicMessagesResponse, + source: CoreEngine, +) -> AnthropicMessagesResponse: + existing_hidden_params: Final = response.get("_hidden_params") + response["_hidden_params"] = execution_hidden_params( + existing_hidden_params if isinstance(existing_hidden_params, dict) else None, + source, + ) + return response + + from .http_handler import get_shared_realtime_ssl_context if TYPE_CHECKING: @@ -2362,8 +2374,21 @@ class BaseLLMHTTPHandler: kwargs=kwargs_for_agentic, ) + selected_response: Final = final_response if final_response is not None else initial_response + if not isinstance(selected_response, dict): + return self._maybe_wrap_in_fake_stream( + selected_response, + logging_obj, + "anthropic_messages", + ) + typed_response: Final = cast(AnthropicMessagesResponse, selected_response) + existing_hidden_params: Final = typed_response.get("_hidden_params") + existing_source: Final = ( + existing_hidden_params.get("core_engine") if isinstance(existing_hidden_params, dict) else None + ) + source: Final = CoreEngine.RUST if existing_source == CoreEngine.RUST.value else CoreEngine.PYTHON return self._maybe_wrap_in_fake_stream( - final_response if final_response is not None else initial_response, + _anthropic_messages_with_core_engine(typed_response, source), logging_obj, "anthropic_messages", ) @@ -2408,7 +2433,7 @@ class BaseLLMHTTPHandler: return None response_obj: Final = cast(AnthropicMessagesResponse, dict(rust_response)) - return response_obj + return _anthropic_messages_with_core_engine(response_obj, CoreEngine.RUST) @staticmethod def _rust_anthropic_messages_fake_stream( @@ -2423,7 +2448,10 @@ class BaseLLMHTTPHandler: ) completion_stream = cast(AsyncIterator[bytes], FakeAnthropicMessagesStreamIterator(response=rust_response)) - hidden_params: Final = AnthropicMessagesStreamHiddenParams(additional_headers={}) + hidden_params: Final = AnthropicMessagesStreamHiddenParams( + additional_headers=execution_additional_headers(None, CoreEngine.RUST), + core_engine=CoreEngine.RUST.value, + ) return AnthropicMessagesStreamingResponse( completion_stream=completion_stream, hidden_params=hidden_params, @@ -6484,11 +6512,12 @@ class BaseLLMHTTPHandler: if _rust_responses_websocket_enabled(custom_llm_provider, litellm_params): from litellm.rust_bridge import responses_websocket as rust_responses_websocket - rust_backend: Final = await rust_responses_websocket.connect( + rust_execution: Final = await rust_responses_websocket.connect( url=ws_url, headers={str(key): str(value) for key, value in headers.items()}, timeout=timeout, ) + rust_backend: Final = rust_execution.value if rust_backend is not None: yield rust_backend return diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index b260ec6e06f..e361f68aad6 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -29,6 +29,7 @@ from litellm.llms.base_llm.ocr.transformation import ( ) from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.rust_bridge import ocr as rust_ocr_bridge +from litellm.rust_bridge.runtime import CoreEngine, execution_hidden_params from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client @@ -67,6 +68,24 @@ _RUST_OCR_PROVIDERS: Final = { } +def _with_core_engine(response: OCRResponse, source: CoreEngine) -> OCRResponse: + raw_hidden_params: Final = cast(object, response._hidden_params) + hidden_params: Final = ( + cast(Mapping[str, object], raw_hidden_params) if isinstance(raw_hidden_params, Mapping) else None + ) + response._hidden_params = execution_hidden_params( # pyright: ignore[reportPrivateUsage] # OCRResponse has no public metadata setter + hidden_params, source + ) + return response + + +async def _await_with_core_engine( + response: Coroutine[object, object, OCRResponse], + source: CoreEngine, +) -> OCRResponse: + return _with_core_engine(await response, source) + + def _prepare_ocr_request( model: str, document: Mapping[str, object], @@ -306,7 +325,7 @@ def _run_rust_ocr( ) if rust_response is None: return None - return OCRResponse.model_validate(rust_response) + return _with_core_engine(OCRResponse.model_validate(rust_response), CoreEngine.RUST) async def _run_rust_aocr( @@ -331,7 +350,7 @@ async def _run_rust_aocr( ) if rust_response is None: return None - return OCRResponse.model_validate(rust_response) + return _with_core_engine(OCRResponse.model_validate(rust_response), CoreEngine.RUST) @client @@ -461,7 +480,7 @@ async def aocr( if response is None: raise ValueError(f"Got an unexpected None response from the OCR API: {response}") - return response + return _with_core_engine(response, CoreEngine.PYTHON) except Exception as e: raise litellm.exception_type( model=model, @@ -727,7 +746,9 @@ def ocr( litellm_params=prepared.litellm_params, ) - return response + if asyncio.iscoroutine(response): + return _await_with_core_engine(response, CoreEngine.PYTHON) + return _with_core_engine(response, CoreEngine.PYTHON) except Exception as e: raise litellm.exception_type( model=model, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 05ddef822f1..c631cc7d74a 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1553,6 +1553,16 @@ class ProxyBaseLLMRequestProcessing: logging_obj=litellm_logging_obj, use_logging_obj=read_timing_from_logging_obj, ) + core_engine_value: Final = hidden_params.get("core_engine") + core_engine: Final = ( + core_engine_value + if isinstance(core_engine_value, str) and core_engine_value in ("python", "rust") + else "python" + ) + reserved_core_headers: Final = frozenset({"x-litellm-core", "x-litellm-rust"}) + forwarded_headers: Final = { + key: value for key, value in kwargs.items() if key.lower() not in reserved_core_headers + } cost_breakdown: Final = _get_cost_breakdown_from_logging_obj( litellm_logging_obj=litellm_logging_obj, response_cost=response_cost @@ -1637,7 +1647,9 @@ class ProxyBaseLLMRequestProcessing: str(fastest_response_batch_completion) if fastest_response_batch_completion is not None else None ), "x-litellm-timeout": str(timeout) if timeout is not None else None, - **{k: str(v) for k, v in kwargs.items()}, + **{key: str(value) for key, value in forwarded_headers.items()}, + "x-litellm-core": core_engine, + **({"x-litellm-rust": "true"} if core_engine == "rust" else {}), } if request_data: remaining_tokens_header: Final = get_remaining_tokens_and_requests_from_request_data(request_data) diff --git a/litellm/rust_bridge/chat_completions.py b/litellm/rust_bridge/chat_completions.py index efdb6eaa9c5..5fb89ae5848 100644 --- a/litellm/rust_bridge/chat_completions.py +++ b/litellm/rust_bridge/chat_completions.py @@ -14,7 +14,7 @@ from __future__ import annotations import json from collections.abc import Awaitable, Callable, Mapping, Sequence -from typing import TYPE_CHECKING, Final, Protocol +from typing import TYPE_CHECKING, Final, Protocol, cast import httpx from pydantic import TypeAdapter, ValidationError @@ -28,12 +28,15 @@ from litellm.rust_bridge.bindings import UNSET, NativeBinding, Unset from litellm.rust_bridge.configuration import rust_enabled from litellm.rust_bridge.runtime import ( BridgeErrorContext, + CoreEngine, + ExecutionResult, FallbackMode, RustDeclined, RustHandled, aattempt, ainvoke, attempt, + execution_hidden_params, identity, invoke, ) @@ -51,6 +54,7 @@ RUST_CHAT_COMPLETIONS_PROVIDERS: Final = frozenset({"anthropic", "bedrock"}) # rather than narrowing an unparameterized `Mapping` and typing the result Any. _LITELLM_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + class RustChatCompletions(Protocol): def __call__( self, @@ -265,12 +269,31 @@ def _build_model_response( built: Final = convert_to_model_response_object( response_object=dict(rust_response), # mutable-ok: the converter takes a real dict and rewrites it model_response_object=model_response, + hidden_params=execution_hidden_params(None, CoreEngine.RUST), # mutable-ok: rewritten by the converter ) if not isinstance(built, ModelResponse): raise TypeError(f"expected a ModelResponse from the rust path, got {type(built).__name__}") return built +def _unwrap_execution(result: ExecutionResult[object]) -> object: + value: Final = result.value + if isinstance(value, ModelResponse): + raw_hidden_params: Final = cast( + object, + value._hidden_params, # pyright: ignore[reportPrivateUsage] # ModelResponse has no public metadata getter + ) + hidden_params: Final = ( + cast(Mapping[str, object], raw_hidden_params) # cast-ok: guarded by the mapping check + if isinstance(raw_hidden_params, Mapping) + else None + ) + value._hidden_params = execution_hidden_params( # pyright: ignore[reportPrivateUsage] # ModelResponse has no public metadata setter + hidden_params, result.source + ) + return value + + def chat_completions( *, model: str, @@ -387,12 +410,14 @@ def chat_completions_or_fallback( on_response(rust_response) return _build_model_response(rust_response, model_response) - return invoke( - native_call=native_call, - fallback=python_fallback, - adapt=adapt, - mode=FallbackMode.PYTHON, - context=BridgeErrorContext(route="chat completions", provider=custom_llm_provider or "", model=model), + return _unwrap_execution( + invoke( + native_call=native_call, + fallback=python_fallback, + adapt=adapt, + mode=FallbackMode.PYTHON, + context=BridgeErrorContext(route="chat completions", provider=custom_llm_provider or "", model=model), + ) ) @@ -430,10 +455,12 @@ async def achat_completions_or_fallback( on_response(rust_response) return _build_model_response(rust_response, model_response) - return await ainvoke( - native_call=native_call, - fallback=python_fallback, - adapt=adapt, - mode=FallbackMode.PYTHON, - context=BridgeErrorContext(route="chat completions", provider=custom_llm_provider or "", model=model), + return _unwrap_execution( + await ainvoke( + native_call=native_call, + fallback=python_fallback, + adapt=adapt, + mode=FallbackMode.PYTHON, + context=BridgeErrorContext(route="chat completions", provider=custom_llm_provider or "", model=model), + ) ) diff --git a/litellm/rust_bridge/messages.py b/litellm/rust_bridge/messages.py index e0bdef20dfe..82ff6cabcb7 100644 --- a/litellm/rust_bridge/messages.py +++ b/litellm/rust_bridge/messages.py @@ -105,7 +105,7 @@ def messages( adapt=identity, mode=FallbackMode.PYTHON, context=_context(model, custom_llm_provider), - ) + ).value async def amessages( @@ -119,22 +119,24 @@ async def amessages( timeout: float | httpx.Timeout | None, ) -> dict[str, object] | None: native: Final = load_rust_amessages() - return await ainvoke( - native_call=( - None - if native is None - else lambda: native( - model=model, - body=body, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - timeout_seconds=timeout_to_seconds(timeout), - ) - ), - fallback=async_none, - adapt=identity, - mode=FallbackMode.PYTHON, - context=_context(model, custom_llm_provider), - ) + return ( + await ainvoke( + native_call=( + None + if native is None + else lambda: native( + model=model, + body=body, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + timeout_seconds=timeout_to_seconds(timeout), + ) + ), + fallback=async_none, + adapt=identity, + mode=FallbackMode.PYTHON, + context=_context(model, custom_llm_provider), + ) + ).value diff --git a/litellm/rust_bridge/ocr.py b/litellm/rust_bridge/ocr.py index 355254fb237..401ca3448f9 100644 --- a/litellm/rust_bridge/ocr.py +++ b/litellm/rust_bridge/ocr.py @@ -106,7 +106,7 @@ def ocr( adapt=identity, mode=FallbackMode.PYTHON, context=BridgeErrorContext(route="ocr", provider=custom_llm_provider or "", model=model), - ) + ).value async def aocr( @@ -135,10 +135,12 @@ async def aocr( timeout_seconds=_timeout_to_seconds(timeout), ) ) - return await ainvoke( - native_call=native_call, - fallback=async_none, - adapt=identity, - mode=FallbackMode.PYTHON, - context=BridgeErrorContext(route="ocr", provider=custom_llm_provider or "", model=model), - ) + return ( + await ainvoke( + native_call=native_call, + fallback=async_none, + adapt=identity, + mode=FallbackMode.PYTHON, + context=BridgeErrorContext(route="ocr", provider=custom_llm_provider or "", model=model), + ) + ).value diff --git a/litellm/rust_bridge/responses_websocket.py b/litellm/rust_bridge/responses_websocket.py index 4d505a757c9..4c1c2330f5c 100644 --- a/litellm/rust_bridge/responses_websocket.py +++ b/litellm/rust_bridge/responses_websocket.py @@ -10,6 +10,8 @@ from websockets.exceptions import ConnectionClosedOK from litellm.rust_bridge.bindings import UNSET, NativeBinding, Unset from litellm.rust_bridge.runtime import ( BridgeErrorContext, + CoreEngine, + ExecutionResult, FallbackMode, acall, ainvoke, @@ -36,9 +38,7 @@ class RustResponsesWebSocketConnection(Protocol): ) -> RustResponsesWebSocket: ... -_CONNECTION: Final = NativeBinding[type[RustResponsesWebSocketConnection]]( - "ResponsesWebSocketConnection" -) +_CONNECTION: Final = NativeBinding[type[RustResponsesWebSocketConnection]]("ResponsesWebSocketConnection") def set_rust_responses_websocket( @@ -55,6 +55,7 @@ def load_rust_responses_websocket() -> type[RustResponsesWebSocketConnection] | class _ConnectionAdapter: def __init__(self, connection: RustResponsesWebSocket): self._connection: Final = connection + self.core_engine: Final = CoreEngine.RUST async def send(self, text: str) -> None: await acall( @@ -83,7 +84,7 @@ async def connect( url: str, headers: dict[str, str], timeout: float | httpx.Timeout | None, -) -> _ConnectionAdapter | None: +) -> ExecutionResult[_ConnectionAdapter | None]: connection_type: Final = load_rust_responses_websocket() native_call: Final = ( None diff --git a/litellm/rust_bridge/runtime.py b/litellm/rust_bridge/runtime.py index 0bf60b1aa0b..e27099d4d22 100644 --- a/litellm/rust_bridge/runtime.py +++ b/litellm/rust_bridge/runtime.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from enum import Enum from typing import Final, Generic, NoReturn, TypeAlias, TypeVar, cast @@ -11,12 +11,27 @@ from litellm.rust_bridge.bindings import native_exception_types NativeT = TypeVar("NativeT") ResultT = TypeVar("ResultT") +CORE_ENGINE_HIDDEN_PARAM: Final = "core_engine" +CORE_ENGINE_HEADER: Final = "x-litellm-core" +LEGACY_RUST_HEADER: Final = "x-litellm-rust" + class FallbackMode(Enum): PYTHON = "python" RUST_REQUIRED = "rust_required" +class CoreEngine(str, Enum): + PYTHON = "python" + RUST = "rust" + + +@dataclass(frozen=True, slots=True) +class ExecutionResult(Generic[ResultT]): + value: ResultT + source: CoreEngine + + @dataclass(frozen=True, slots=True) class RustHandled(Generic[ResultT]): value: ResultT @@ -42,6 +57,48 @@ class BridgeErrorContext: model: str +def execution_headers(source: CoreEngine) -> dict[str, str]: + if source is CoreEngine.RUST: + return { # mutable-ok: response adapters require a mutable header dict + CORE_ENGINE_HEADER: source.value, + LEGACY_RUST_HEADER: "true", + } + return {CORE_ENGINE_HEADER: source.value} # mutable-ok: response adapters require a mutable header dict + + +def execution_additional_headers( + additional_headers: Mapping[str, object] | None, + source: CoreEngine, +) -> dict[str, str]: + existing: Final = additional_headers or {} # mutable-ok: empty default is local and never mutated + reserved: Final = frozenset({CORE_ENGINE_HEADER, LEGACY_RUST_HEADER}) + preserved: Final = { # mutable-ok: response adapters require a mutable header dict + str(name): str(value) for name, value in existing.items() if str(name).lower() not in reserved + } + return { # mutable-ok: response adapters require a mutable header dict + **preserved, + **execution_headers(source), + } + + +def execution_hidden_params( + hidden_params: Mapping[str, object] | None, + source: CoreEngine, +) -> dict[str, object]: + existing: Final = hidden_params or {} # mutable-ok: empty default is local and never mutated + raw_headers: Final = existing.get("additional_headers") + additional_headers: Final[Mapping[str, object]] = ( + cast(Mapping[str, object], raw_headers) # cast-ok: the isinstance check validates the mapping boundary + if isinstance(raw_headers, Mapping) + else {} # mutable-ok: empty default is local and never mutated + ) + return { # mutable-ok: response objects require mutable hidden params + **existing, + CORE_ENGINE_HIDDEN_PARAM: source.value, + "additional_headers": execution_additional_headers(additional_headers, source), + } + + def invoke( *, native_call: Callable[[], NativeT] | None, @@ -49,12 +106,12 @@ def invoke( adapt: Callable[[NativeT], ResultT], mode: FallbackMode, context: BridgeErrorContext, -) -> ResultT: +) -> ExecutionResult[ResultT]: result: Final = attempt(native_call=native_call, adapt=adapt, context=context) if isinstance(result, RustHandled): - return result.value + return ExecutionResult(value=result.value, source=CoreEngine.RUST) if mode is FallbackMode.PYTHON: - return fallback() + return ExecutionResult(value=fallback(), source=CoreEngine.PYTHON) _raise_required(result, context) @@ -65,12 +122,12 @@ async def ainvoke( adapt: Callable[[NativeT], ResultT], mode: FallbackMode, context: BridgeErrorContext, -) -> ResultT: +) -> ExecutionResult[ResultT]: result: Final = await aattempt(native_call=native_call, adapt=adapt, context=context) if isinstance(result, RustHandled): - return result.value + return ExecutionResult(value=result.value, source=CoreEngine.RUST) if mode is FallbackMode.PYTHON: - return await fallback() + return ExecutionResult(value=await fallback(), source=CoreEngine.PYTHON) _raise_required(result, context) @@ -164,12 +221,16 @@ def _raise_upstream(error: BaseException, context: BridgeErrorContext) -> NoRetu message_value: Final = args[1] if len(args) > 1 else str(error) status: Final = status_value if isinstance(status_value, int) else 0 message: Final = message_value if isinstance(message_value, str) else str(message_value) - raise APIError( + api_error: Final = APIError( status_code=status or 500, message=f"litellm rust {context.route}: {message}", llm_provider=context.provider, model=context.model, - ) from error + ) + api_error.headers = execution_headers( # pyright: ignore[reportAttributeAccessIssue] # proxy reads exception headers + CoreEngine.RUST + ) + raise api_error from error def identity(value: ResultT) -> ResultT: diff --git a/litellm/rust_bridge/transcription.py b/litellm/rust_bridge/transcription.py index bf1e3fb60d8..295b0d4ad01 100644 --- a/litellm/rust_bridge/transcription.py +++ b/litellm/rust_bridge/transcription.py @@ -102,7 +102,7 @@ def transcription( adapt=identity, mode=FallbackMode.RUST_REQUIRED, context=BridgeErrorContext(route="audio transcription", provider=custom_llm_provider or "", model=model), - ) + ).value async def atranscription( @@ -131,10 +131,12 @@ async def atranscription( timeout_seconds=timeout_to_seconds(timeout), ) ) - return await ainvoke( - native_call=native_call, - fallback=async_none, - adapt=identity, - mode=FallbackMode.RUST_REQUIRED, - context=BridgeErrorContext(route="audio transcription", provider=custom_llm_provider or "", model=model), - ) + return ( + await ainvoke( + native_call=native_call, + fallback=async_none, + adapt=identity, + mode=FallbackMode.RUST_REQUIRED, + context=BridgeErrorContext(route="audio transcription", provider=custom_llm_provider or "", model=model), + ) + ).value diff --git a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py index 362d3a5f5a0..f1a4d584ab6 100644 --- a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py +++ b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py @@ -431,7 +431,11 @@ async def test_fake_stream_wraps_rust_response_as_anthropic_sse(): response = cast(AnthropicMessagesResponse, dict(FAKE_MESSAGES_RESPONSE)) stream = BaseLLMHTTPHandler._rust_anthropic_messages_fake_stream(response) - assert stream._hidden_params["additional_headers"] == {} + assert stream._hidden_params["core_engine"] == "rust" + assert stream._hidden_params["additional_headers"] == { + "x-litellm-core": "rust", + "x-litellm-rust": "true", + } chunks = [chunk async for chunk in stream] joined = b"".join(chunks) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index e8f3ea3a0a9..395b151b224 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -450,6 +450,11 @@ def test_run_rust_ocr_prepares_request_and_wraps_response(): assert isinstance(response, OCRResponse) assert response.pages[0].markdown == "hello world" + assert response._hidden_params["core_engine"] == "rust" + assert response._hidden_params["additional_headers"] == { + "x-litellm-core": "rust", + "x-litellm-rust": "true", + } assert bridge.calls[0] == { "model": "mistral-ocr-latest", "document": DOCUMENT, @@ -643,6 +648,7 @@ def test_ocr_routes_to_rust_when_enabled(fake_bridge): assert isinstance(response, OCRResponse) assert response.pages[0].markdown == "hello world" + assert response._hidden_params["core_engine"] == "rust" assert len(fake_bridge.calls) == 1 call = fake_bridge.calls[0] assert call["model"] == "mistral-ocr-latest" @@ -792,6 +798,8 @@ def test_ocr_falls_back_to_python_when_bridge_unavailable(monkeypatch): assert captured.get("called") is True # Python path was used assert isinstance(response, OCRResponse) + assert response._hidden_params["core_engine"] == "python" + assert response._hidden_params["additional_headers"] == {"x-litellm-core": "python"} def test_ocr_unsupported_provider_skips_rust(monkeypatch): @@ -810,6 +818,7 @@ def test_ocr_unsupported_provider_skips_rust(monkeypatch): ) assert isinstance(response, OCRResponse) + assert response._hidden_params["core_engine"] == "python" assert bridge.calls == [] diff --git a/tests/test_litellm/responses/test_rust_bridge_websocket.py b/tests/test_litellm/responses/test_rust_bridge_websocket.py index 9aa9c4acd3d..12f75bc32f0 100644 --- a/tests/test_litellm/responses/test_rust_bridge_websocket.py +++ b/tests/test_litellm/responses/test_rust_bridge_websocket.py @@ -78,14 +78,13 @@ async def test_adapter_raises_clean_close_when_rust_connection_ends() -> None: async def test_bridge_unavailable_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) - assert ( - await responses_websocket.connect( - url="wss://example.test/responses", - headers={}, - timeout=None, - ) - is None + result = await responses_websocket.connect( + url="wss://example.test/responses", + headers={}, + timeout=None, ) + assert result.value is None + assert result.source is responses_websocket.CoreEngine.PYTHON @pytest.mark.asyncio @@ -94,13 +93,16 @@ async def test_enabled_bridge_connects_and_adapts_socket( ) -> None: responses_websocket.set_rust_responses_websocket(connection=_FakeNativeBridge) - connection = await responses_websocket.connect( + result = await responses_websocket.connect( url="wss://example.test/responses", headers={"Authorization": "Bearer key"}, timeout=1.0, ) + assert result.source is responses_websocket.CoreEngine.RUST + connection = result.value assert connection is not None + assert connection.core_engine is responses_websocket.CoreEngine.RUST await connection.send("response.create") assert await connection.recv() == "response.completed" await connection.close() diff --git a/tests/test_litellm/rust_bridge/test_chat_completions.py b/tests/test_litellm/rust_bridge/test_chat_completions.py index a7c8570e78d..fd114c0b9a3 100644 --- a/tests/test_litellm/rust_bridge/test_chat_completions.py +++ b/tests/test_litellm/rust_bridge/test_chat_completions.py @@ -264,6 +264,11 @@ class TestSyncCall: assert result.usage.completion_tokens == 4 assert result.usage.total_tokens == 15 assert result.id == original_id, "the rust path must keep the chatcmpl id litellm already minted" + assert result._hidden_params["core_engine"] == "rust" + assert result._hidden_params["additional_headers"] == { + "x-litellm-core": "rust", + "x-litellm-rust": "true", + } def test_passes_the_timeout_through_as_seconds(self): native = _RecordingCall() @@ -291,6 +296,8 @@ class TestSyncCall: ) assert result is fallback_response + assert result._hidden_params["core_engine"] == "python" + assert result._hidden_params["additional_headers"] == {"x-litellm-core": "python"} class TestAsyncCall: @@ -300,6 +307,7 @@ class TestAsyncCall: result = await bridge.achat_completions_or_fallback(**_async_call_kwargs(ModelResponse())) assert result is not None assert result.choices[0].message.content == "hello from rust" + assert result._hidden_params["core_engine"] == "rust" @pytest.mark.asyncio async def test_falls_back_when_the_bridge_is_unavailable(self, monkeypatch): diff --git a/tests/test_litellm/rust_bridge/test_runtime.py b/tests/test_litellm/rust_bridge/test_runtime.py index b0fa510069b..9d0fd6d2098 100644 --- a/tests/test_litellm/rust_bridge/test_runtime.py +++ b/tests/test_litellm/rust_bridge/test_runtime.py @@ -44,7 +44,8 @@ def test_invoke_tags_native_decline_before_running_fallback() -> None: context=context(), ) - assert value == "fallback" + assert value.value == "fallback" + assert value.source is runtime.CoreEngine.PYTHON assert calls == ["rust", "python"] @@ -62,6 +63,10 @@ def test_invoke_translates_upstream_without_fallback() -> None: ) assert caught.value.status_code == 429 + assert caught.value.headers == { + "x-litellm-core": "rust", + "x-litellm-rust": "true", + } @pytest.mark.asyncio @@ -72,16 +77,38 @@ async def test_ainvoke_handles_native_success() -> None: async def fallback() -> str: pytest.fail("fallback must not run") - assert ( - await runtime.ainvoke( - native_call=native, - fallback=fallback, - adapt=str, - mode=runtime.FallbackMode.PYTHON, - context=context(), - ) - == "3" + result = await runtime.ainvoke( + native_call=native, + fallback=fallback, + adapt=str, + mode=runtime.FallbackMode.PYTHON, + context=context(), ) + assert result.value == "3" + assert result.source is runtime.CoreEngine.RUST + + +def test_execution_hidden_params_overwrites_reserved_provider_headers() -> None: + hidden_params = runtime.execution_hidden_params( + { + "provider": "anthropic", + "additional_headers": { + "request-id": "req-1", + "X-LiteLLM-Core": "spoofed", + "x-litellm-rust": "spoofed", + }, + }, + runtime.CoreEngine.PYTHON, + ) + + assert hidden_params == { + "provider": "anthropic", + "core_engine": "python", + "additional_headers": { + "request-id": "req-1", + "x-litellm-core": "python", + }, + } def test_required_mode_rejects_unavailable_bridge() -> None: diff --git a/tests/test_litellm/test_audio_transcription_rust_bridge.py b/tests/test_litellm/test_audio_transcription_rust_bridge.py index 92e86f48c31..f9fdfe84c25 100644 --- a/tests/test_litellm/test_audio_transcription_rust_bridge.py +++ b/tests/test_litellm/test_audio_transcription_rust_bridge.py @@ -133,6 +133,11 @@ def test_bedrock_transcription_uses_rust_only_path() -> None: rust_bridge.configure_rust_transcription(transcription=None, atranscription=None) assert response.text == "rust" + assert response._hidden_params["core_engine"] == "rust" + assert response._hidden_params["additional_headers"] == { + "x-litellm-core": "rust", + "x-litellm-rust": "true", + } @pytest.mark.asyncio @@ -150,3 +155,8 @@ async def test_bedrock_atranscription_uses_rust_only_path() -> None: rust_bridge.configure_rust_transcription(transcription=None, atranscription=None) assert response.text == "rust" + assert response._hidden_params["core_engine"] == "rust" + assert response._hidden_params["additional_headers"] == { + "x-litellm-core": "rust", + "x-litellm-rust": "true", + } From 759740135aeeb077971b6fd8906f427f61b07240 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 06:59:53 -0700 Subject: [PATCH 61/67] feat(rust): add typed streaming transport --- litellm-rust/Cargo.lock | 1 + litellm-rust/crates/core/Cargo.toml | 1 + litellm-rust/crates/core/src/lib.rs | 1 + litellm-rust/crates/core/src/streaming.rs | 636 ++++++++++++++++++++++ 4 files changed, 639 insertions(+) create mode 100644 litellm-rust/crates/core/src/streaming.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 33f6bb5a87e..b2f788beb37 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1428,6 +1428,7 @@ dependencies = [ "aws-sigv4", "aws-smithy-runtime-api", "aws-types", + "futures-util", "rand 0.8.7", "reqwest", "serde", diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index ab8050734f2..2b7c76a9bec 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +futures-util.workspace = true rand.workspace = true reqwest.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index 0e18d24e5d8..3a651111f8d 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -12,5 +12,6 @@ pub mod realtime; pub mod responses; pub mod router; pub mod routing_utils; +pub mod streaming; pub use error::Error; diff --git a/litellm-rust/crates/core/src/streaming.rs b/litellm-rust/crates/core/src/streaming.rs new file mode 100644 index 00000000000..811ff201200 --- /dev/null +++ b/litellm-rust/crates/core/src/streaming.rs @@ -0,0 +1,636 @@ +use std::collections::VecDeque; +use std::marker::PhantomData; +use std::pin::Pin; +use std::time::Duration; + +use crate::error::Error; +use futures_util::future::BoxFuture; +use futures_util::{Stream, StreamExt, stream}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +pub type EventStream = Pin> + Send + 'static>>; +pub type ProviderChunkStream = + Pin> + Send + 'static>>; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StreamTransport { + Http, + WebSocket, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StreamProviderId { + Anthropic, + AzureAi, + BedrockConverse, + OpenAi, +} + +impl TryFrom<&str> for StreamProviderId { + type Error = Error; + + fn try_from(value: &str) -> Result { + match value { + "anthropic" => Ok(Self::Anthropic), + "azure_ai" => Ok(Self::AzureAi), + "bedrock" | "bedrock_converse" => Ok(Self::BedrockConverse), + "openai" => Ok(Self::OpenAi), + _ => Err(Error::InvalidProvider(value.to_string())), + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct JsonObject(pub Map); + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Header { + pub name: String, + pub value: String, +} + +#[derive(Clone, Default, PartialEq)] +pub struct ProviderCredentials { + api_key: Option, + aws_access_key_id: Option, + aws_secret_access_key: Option, + aws_session_token: Option, +} + +impl ProviderCredentials { + pub fn new( + api_key: Option, + aws_access_key_id: Option, + aws_secret_access_key: Option, + aws_session_token: Option, + ) -> Self { + Self { + api_key, + aws_access_key_id, + aws_secret_access_key, + aws_session_token, + } + } + + pub fn api_key(&self) -> Option<&str> { + self.api_key.as_deref() + } + + pub fn aws_access_key_id(&self) -> Option<&str> { + self.aws_access_key_id.as_deref() + } + + pub fn aws_secret_access_key(&self) -> Option<&str> { + self.aws_secret_access_key.as_deref() + } + + pub fn aws_session_token(&self) -> Option<&str> { + self.aws_session_token.as_deref() + } +} + +/// ```compile_fail +/// fn assert_serialize() {} +/// assert_serialize::(); +/// assert_serialize::(); +/// assert_serialize::(); +/// ``` +/// +/// ```compile_fail +/// use litellm_core::streaming::{JsonObject, ProviderCredentials, StreamProviderId, StreamTarget}; +/// let mut target = StreamTarget::new( +/// StreamProviderId::OpenAi, +/// ProviderCredentials::default(), +/// None, +/// ); +/// target.metadata = JsonObject::default(); +/// ``` +#[derive(Clone, PartialEq)] +pub struct StreamTarget { + provider: StreamProviderId, + credentials: ProviderCredentials, + api_base: Option, +} + +impl StreamTarget { + pub fn new( + provider: StreamProviderId, + credentials: ProviderCredentials, + api_base: Option, + ) -> Self { + Self { + provider, + credentials, + api_base, + } + } + + pub fn provider(&self) -> StreamProviderId { + self.provider + } + + pub fn credentials(&self) -> &ProviderCredentials { + &self.credentials + } + + pub fn api_base(&self) -> Option<&str> { + self.api_base.as_deref() + } +} + +#[derive(Clone, Default, PartialEq)] +pub struct StreamTransportOptions { + forwarded_headers: Vec
, + timeout: Option, +} + +impl StreamTransportOptions { + pub fn new(forwarded_headers: Vec
, timeout: Option) -> Self { + Self { + forwarded_headers, + timeout, + } + } + + pub fn forwarded_headers(&self) -> &[Header] { + &self.forwarded_headers + } + + pub fn timeout(&self) -> Option { + self.timeout + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StreamMetadata { + pub status_code: u16, + pub provider: StreamProviderId, + pub transport: StreamTransport, + pub response_headers: Vec
, +} + +pub struct OpenedStream { + pub metadata: StreamMetadata, + pub events: EventStream, +} + +pub struct OpenedWireStream { + pub metadata: StreamMetadata, + pub chunks: ProviderChunkStream, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProviderStreamChunk(Vec); + +impl ProviderStreamChunk { + pub fn new(bytes: impl Into>) -> Self { + Self(bytes.into()) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +pub trait StreamDecoder: Send + 'static { + type WireEvent: Send + 'static; + + fn push(&mut self, chunk: ProviderStreamChunk) -> Result, Error>; + + fn finish(&mut self) -> Result, Error> { + Ok(Vec::new()) + } +} + +pub trait StreamProvider: Send + Sync + 'static { + type PreparedRequest: Send + 'static; + type WireEvent: Send + 'static; + type Decoder: StreamDecoder; + + fn transform_request(&self, request: R) -> Result; + + fn call( + &'static self, + request: Self::PreparedRequest, + ) -> BoxFuture<'static, Result>; + + fn decoder(&self) -> Self::Decoder; + + fn normalize(&self, event: Self::WireEvent) -> Result, Error>; +} + +struct PipelineState +where + D: StreamDecoder, +{ + provider: &'static P, + decoder: D, + chunks: ProviderChunkStream, + pending: VecDeque>, + finished: bool, + request: PhantomData, +} + +pub async fn open_provider_stream( + provider: &'static P, + request: R, +) -> Result, Error> +where + P: StreamProvider, + R: Send + 'static, + E: Send + 'static, +{ + let prepared = provider.transform_request(request)?; + let opened = provider.call(prepared).await?; + let state = PipelineState { + provider, + decoder: provider.decoder(), + chunks: opened.chunks, + pending: VecDeque::new(), + finished: false, + request: PhantomData, + }; + let events = stream::unfold(state, |mut state| async move { + loop { + if let Some(event) = state.pending.pop_front() { + return Some((event, state)); + } + if state.finished { + return None; + } + match state.chunks.next().await { + Some(Ok(chunk)) => match state.decoder.push(chunk) { + Ok(events) => queue_normalized(&mut state, events), + Err(error) => { + state.finished = true; + return Some((Err(error), state)); + } + }, + Some(Err(error)) => { + state.finished = true; + return Some((Err(error), state)); + } + None => { + state.finished = true; + match state.decoder.finish() { + Ok(events) => queue_normalized(&mut state, events), + Err(error) => return Some((Err(error), state)), + } + } + } + } + }); + Ok(OpenedStream { + metadata: opened.metadata, + events: Box::pin(events), + }) +} + +fn queue_normalized(state: &mut PipelineState, events: Vec) +where + P: StreamProvider, + D: StreamDecoder>::WireEvent>, +{ + for event in events { + match state.provider.normalize(event) { + Ok(normalized) => state.pending.extend(normalized.into_iter().map(Ok)), + Err(error) => { + state.pending.push_back(Err(error)); + state.finished = true; + return; + } + } + } +} + +#[cfg(test)] +mod tests { + use std::pin::Pin; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + use std::task::{Context, Poll}; + + use futures_util::future::FutureExt; + + use super::*; + + struct FakeRequest; + struct PreparedRequest; + + struct FakeProvider { + calls: Arc>>, + } + + struct FakeDecoder { + calls: Arc>>, + pending: String, + } + + #[derive(Clone, Copy)] + enum FailurePoint { + Transform, + Call, + Chunk, + Push, + Finish, + Normalize, + } + + struct FailureProvider(FailurePoint); + + struct FailureDecoder(FailurePoint); + + impl StreamDecoder for FailureDecoder { + type WireEvent = String; + + fn push(&mut self, chunk: ProviderStreamChunk) -> Result, Error> { + if matches!(self.0, FailurePoint::Push) { + return Err(Error::InvalidResponse("decoder push failed".to_string())); + } + Ok(vec![ + String::from_utf8(chunk.0).expect("test chunk should be UTF-8"), + ]) + } + + fn finish(&mut self) -> Result, Error> { + if matches!(self.0, FailurePoint::Finish) { + return Err(Error::InvalidResponse("decoder finish failed".to_string())); + } + Ok(Vec::new()) + } + } + + impl StreamProvider for FailureProvider { + type PreparedRequest = PreparedRequest; + type WireEvent = String; + type Decoder = FailureDecoder; + + fn transform_request(&self, _request: FakeRequest) -> Result { + if matches!(self.0, FailurePoint::Transform) { + return Err(Error::InvalidRequest("transform failed".to_string())); + } + Ok(PreparedRequest) + } + + fn call( + &'static self, + _request: Self::PreparedRequest, + ) -> BoxFuture<'static, Result> { + async move { + if matches!(self.0, FailurePoint::Call) { + return Err(Error::Network("open failed".to_string())); + } + let chunks: ProviderChunkStream = match self.0 { + FailurePoint::Chunk => Box::pin(stream::iter([ + Err(Error::Network("source failed".to_string())), + Ok(ProviderStreamChunk::new("ignored")), + ])), + FailurePoint::Finish => Box::pin(stream::empty()), + _ => Box::pin(stream::iter([Ok(ProviderStreamChunk::new("event"))])), + }; + Ok(OpenedWireStream { + metadata: StreamMetadata { + status_code: 200, + provider: StreamProviderId::OpenAi, + transport: StreamTransport::Http, + response_headers: Vec::new(), + }, + chunks, + }) + } + .boxed() + } + + fn decoder(&self) -> Self::Decoder { + FailureDecoder(self.0) + } + + fn normalize(&self, event: Self::WireEvent) -> Result, Error> { + if matches!(self.0, FailurePoint::Normalize) { + return Err(Error::InvalidResponse("normalize failed".to_string())); + } + Ok(vec![event]) + } + } + + struct PendingUntilDropped(Arc); + + impl Stream for PendingUntilDropped { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } + } + + impl Drop for PendingUntilDropped { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } + } + + struct PendingProvider(Arc); + + impl StreamProvider for PendingProvider { + type PreparedRequest = PreparedRequest; + type WireEvent = String; + type Decoder = FailureDecoder; + + fn transform_request(&self, _request: FakeRequest) -> Result { + Ok(PreparedRequest) + } + + fn call( + &'static self, + _request: Self::PreparedRequest, + ) -> BoxFuture<'static, Result> { + async move { + Ok(OpenedWireStream { + metadata: StreamMetadata { + status_code: 200, + provider: StreamProviderId::OpenAi, + transport: StreamTransport::Http, + response_headers: Vec::new(), + }, + chunks: Box::pin(PendingUntilDropped(self.0.clone())), + }) + } + .boxed() + } + + fn decoder(&self) -> Self::Decoder { + FailureDecoder(FailurePoint::Push) + } + + fn normalize(&self, event: Self::WireEvent) -> Result, Error> { + Ok(vec![event]) + } + } + + impl StreamDecoder for FakeDecoder { + type WireEvent = String; + + fn push(&mut self, chunk: ProviderStreamChunk) -> Result, Error> { + self.calls.lock().expect("call log").push("decode"); + self.pending + .push_str(std::str::from_utf8(chunk.as_bytes()).expect("test utf-8")); + let mut parts = self + .pending + .split('|') + .map(str::to_string) + .collect::>(); + self.pending = parts.pop().expect("split always returns one item"); + Ok(parts) + } + + fn finish(&mut self) -> Result, Error> { + if self.pending.is_empty() { + return Ok(Vec::new()); + } + Ok(vec![std::mem::take(&mut self.pending)]) + } + } + + impl StreamProvider for FakeProvider { + type PreparedRequest = PreparedRequest; + type WireEvent = String; + type Decoder = FakeDecoder; + + fn transform_request(&self, _request: FakeRequest) -> Result { + self.calls.lock().expect("call log").push("transform"); + Ok(PreparedRequest) + } + + fn call( + &'static self, + _request: Self::PreparedRequest, + ) -> BoxFuture<'static, Result> { + self.calls.lock().expect("call log").push("call"); + async move { + Ok(OpenedWireStream { + metadata: StreamMetadata { + status_code: 200, + provider: StreamProviderId::Anthropic, + transport: StreamTransport::Http, + response_headers: vec![Header { + name: "x-test".to_string(), + value: "ready".to_string(), + }], + }, + chunks: Box::pin(stream::iter([ + Ok(ProviderStreamChunk::new(b"one|tw".to_vec())), + Ok(ProviderStreamChunk::new(b"o|three".to_vec())), + ])), + }) + } + .boxed() + } + + fn decoder(&self) -> Self::Decoder { + FakeDecoder { + calls: self.calls.clone(), + pending: String::new(), + } + } + + fn normalize(&self, event: Self::WireEvent) -> Result, Error> { + self.calls.lock().expect("call log").push("normalize"); + Ok(vec![event.to_uppercase()]) + } + } + + #[tokio::test] + async fn fake_provider_proves_pipeline_order_and_fragmentation() { + let calls = Arc::new(Mutex::new(Vec::new())); + let provider = Box::leak(Box::new(FakeProvider { + calls: calls.clone(), + })); + let mut opened = open_provider_stream(provider, FakeRequest) + .await + .expect("stream opens"); + let events = opened + .events + .by_ref() + .collect::>() + .await + .into_iter() + .collect::, _>>() + .expect("events normalize"); + + assert_eq!(events, ["ONE", "TWO", "THREE"]); + assert_eq!(opened.metadata.response_headers[0].name, "x-test"); + assert_eq!( + *calls.lock().expect("call log"), + [ + "transform", + "call", + "decode", + "normalize", + "decode", + "normalize", + "normalize", + ] + ); + } + + #[tokio::test] + async fn request_and_open_failures_stop_before_a_stream_is_returned() { + for (point, expected) in [ + (FailurePoint::Transform, "invalid request: transform failed"), + (FailurePoint::Call, "upstream network error: open failed"), + ] { + let provider = Box::leak(Box::new(FailureProvider(point))); + let error = match open_provider_stream(provider, FakeRequest).await { + Ok(_) => panic!("failure should prevent the stream from opening"), + Err(error) => error, + }; + assert_eq!(error.to_string(), expected); + } + } + + #[tokio::test] + async fn pipeline_failures_are_emitted_once_and_then_terminate() { + for (point, expected) in [ + (FailurePoint::Chunk, "upstream network error: source failed"), + (FailurePoint::Push, "invalid response: decoder push failed"), + ( + FailurePoint::Finish, + "invalid response: decoder finish failed", + ), + ( + FailurePoint::Normalize, + "invalid response: normalize failed", + ), + ] { + let provider = Box::leak(Box::new(FailureProvider(point))); + let mut opened = open_provider_stream(provider, FakeRequest) + .await + .expect("stream should open before its terminal failure"); + let error = opened + .events + .next() + .await + .expect("stream should emit its error") + .expect_err("first event should be the configured failure"); + assert_eq!(error.to_string(), expected); + assert!(opened.events.next().await.is_none()); + } + } + + #[tokio::test] + async fn dropping_the_event_stream_drops_the_provider_chunk_stream() { + let dropped = Arc::new(AtomicBool::new(false)); + let provider = Box::leak(Box::new(PendingProvider(dropped.clone()))); + let opened = open_provider_stream(provider, FakeRequest) + .await + .expect("stream should open"); + + drop(opened.events); + + assert!(dropped.load(Ordering::SeqCst)); + } +} From 71b65063105df17b6e96508d5d8746fb25efbf6a Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 07:03:36 -0700 Subject: [PATCH 62/67] feat(rust): define typed streaming API contracts --- .../crates/core/src/chat_completions/mod.rs | 51 ++++- .../src/chat_completions/transformation.rs | 12 ++ .../crates/core/src/chat_completions/types.rs | 181 ++++++++++++++++ litellm-rust/crates/core/src/messages/mod.rs | 52 ++++- .../core/src/messages/transformation.rs | 16 +- .../crates/core/src/messages/types.rs | 71 +++++++ litellm-rust/crates/core/src/responses/mod.rs | 78 +++++++ .../crates/core/src/responses/types.rs | 194 +++++++++++++++++- .../crates/core/src/responses/websocket.rs | 14 +- 9 files changed, 663 insertions(+), 6 deletions(-) diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index 0d009d36d16..283fccaebe6 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -16,11 +16,14 @@ pub mod response_utils; pub mod transformation; pub mod types; +use crate::streaming::OpenedStream; use serde_json::{Map, Value}; use handler::execute_chat_completions_provider_call; use prepare::{parse_messages, prepare_chat_completions_call, resolve_provider_config}; -use types::{ChatCompletionsRequest, ChatCompletionsResponse}; +use types::{ + ChatCompletionsRequest, ChatCompletionsResponse, ChatCompletionsStreamRequest, ChatStreamEvent, +}; pub async fn chat_completions( request: ChatCompletionsRequest<'_>, @@ -54,5 +57,51 @@ pub fn chat_completions_decline_reason( .map(|reason| reason.0) } +pub async fn chat_completions_stream( + _request: ChatCompletionsStreamRequest, +) -> Result, Error> { + Err(crate::Error::Unsupported( + "chat completions streaming provider registration", + )) +} + #[cfg(test)] mod tests; + +#[cfg(test)] +mod stream_entrypoint_tests { + use serde_json::json; + + use super::*; + use crate::Error; + use crate::streaming::{ + ProviderCredentials, StreamProviderId, StreamTarget, StreamTransportOptions, + }; + + #[tokio::test] + async fn typed_stream_declines_until_a_provider_is_registered() { + let body = serde_json::from_value(json!({ + "model": "claude-sonnet", + "messages": [{"role": "user", "content": "hello"}], + "stream": true + })) + .expect("valid chat stream request"); + let result = chat_completions_stream(ChatCompletionsStreamRequest { + body, + target: StreamTarget::new( + StreamProviderId::Anthropic, + ProviderCredentials::default(), + None, + ), + transport: StreamTransportOptions::default(), + }) + .await; + + assert!(matches!( + result, + Err(Error::Unsupported( + "chat completions streaming provider registration" + )) + )); + } +} diff --git a/litellm-rust/crates/core/src/chat_completions/transformation.rs b/litellm-rust/crates/core/src/chat_completions/transformation.rs index a0868209305..aec28c68886 100644 --- a/litellm-rust/crates/core/src/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/chat_completions/transformation.rs @@ -5,6 +5,18 @@ use super::types::{ ChatCompletionsResponse, ChatMessage, ChatMessageContent, ProviderChatRequestData, ProviderChatResponseData, }; +use super::types::{ChatCompletionsStreamRequest, ChatStreamEvent}; +use crate::streaming::StreamProvider; + +pub trait ChatCompletionsStreamProvider: + StreamProvider +{ +} + +impl ChatCompletionsStreamProvider for T where + T: StreamProvider +{ +} /// How the upstream call is authenticated. API-key strategies are resolved in /// `prepare`; SigV4 needs the serialized body, so the handler signs it. diff --git a/litellm-rust/crates/core/src/chat_completions/types.rs b/litellm-rust/crates/core/src/chat_completions/types.rs index 35dd543a986..ff0273604fb 100644 --- a/litellm-rust/crates/core/src/chat_completions/types.rs +++ b/litellm-rust/crates/core/src/chat_completions/types.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use super::transformation::{ChatCompletionsAuth, ChatCompletionsProviderConfig}; +use crate::streaming::{JsonObject, StreamTarget, StreamTransportOptions}; /// A `/chat/completions` call as it crosses into the core. /// @@ -110,3 +111,183 @@ pub struct ChatCompletionsResponse { pub choices: Vec, pub usage: ChatCompletionsUsage, } + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ChatStreamRole { + Assistant, + Developer, + Function, + System, + Tool, + User, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ChatStreamMessageContent { + Text(String), + Parts(Vec), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatStreamMessage { + pub role: ChatStreamRole, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ChatStreamStop { + One(String), + Many(Vec), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ChatStreamStringOrObject { + Name(String), + Definition(JsonObject), +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionsStreamParameters { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_p: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_completion_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream_options: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tools: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub response_format: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionsStreamRequestBody { + pub model: String, + pub messages: Vec, + #[serde(flatten)] + pub parameters: ChatCompletionsStreamParameters, +} + +pub struct ChatCompletionsStreamRequest { + pub body: ChatCompletionsStreamRequestBody, + pub target: StreamTarget, + pub transport: StreamTransportOptions, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatStreamToolFunctionChunk { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub arguments: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatStreamToolCallChunk { + pub id: Option, + #[serde(rename = "type")] + pub tool_type: String, + pub function: ChatStreamToolFunctionChunk, + pub index: u64, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatStreamUsage { + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub total_tokens: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_tokens_details: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completion_tokens_details: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatStreamEvent { + pub text: String, + pub tool_use: Option, + pub is_finished: bool, + pub finish_reason: String, + pub usage: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub index: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option, +} + +#[cfg(test)] +mod stream_contract_tests { + use super::*; + + #[test] + fn request_uses_public_chat_completion_parameter_names() { + let request: ChatCompletionsStreamRequestBody = serde_json::from_value(serde_json::json!({ + "model": "claude-sonnet", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 32, + "stream": true, + "tool_choice": "auto" + })) + .expect("public request shape"); + + assert_eq!(request.parameters.max_tokens, Some(32)); + assert_eq!(request.parameters.stream, Some(true)); + assert!(matches!( + request.parameters.tool_choice, + Some(ChatStreamStringOrObject::Name(ref value)) if value == "auto" + )); + } + + #[test] + fn event_matches_python_generic_streaming_chunk_shape() { + let event = ChatStreamEvent { + text: "hello".to_string(), + tool_use: None, + is_finished: false, + finish_reason: String::new(), + usage: None, + index: Some(0), + provider_specific_fields: None, + }; + + assert_eq!( + serde_json::to_value(event).expect("serializable event"), + serde_json::json!({ + "text": "hello", + "tool_use": null, + "is_finished": false, + "finish_reason": "", + "usage": null, + "index": 0 + }) + ); + } +} diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index ee2877e61fc..f9a70a991d8 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -14,10 +14,13 @@ mod handler; mod prepare; pub mod transformation; pub mod types; +use crate::streaming::OpenedStream; use handler::{execute_messages_provider_call, execute_messages_provider_stream}; use prepare::prepare_messages_call; -use types::{AnthropicMessagesResponse, MessagesRequest}; +use types::{ + AnthropicMessagesResponse, MessagesRequest, MessagesStreamEvent, MessagesStreamRequest, +}; pub async fn messages(request: MessagesRequest<'_>) -> Result { execute_messages_provider_call(prepare_messages_call(request)?).await @@ -27,5 +30,52 @@ pub async fn messages_stream(request: MessagesRequest<'_>) -> Result Result, Error> { + Err(crate::Error::Unsupported( + "messages event streaming provider registration", + )) +} + #[cfg(test)] mod tests; + +#[cfg(test)] +mod stream_entrypoint_tests { + use serde_json::json; + + use super::*; + use crate::Error; + use crate::streaming::{ + ProviderCredentials, StreamProviderId, StreamTarget, StreamTransportOptions, + }; + + #[tokio::test] + async fn typed_event_stream_declines_until_a_provider_is_registered() { + let body = serde_json::from_value(json!({ + "model": "claude-sonnet", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 32, + "stream": true + })) + .expect("valid Messages stream request"); + let result = messages_event_stream(MessagesStreamRequest { + body, + target: StreamTarget::new( + StreamProviderId::Anthropic, + ProviderCredentials::default(), + None, + ), + transport: StreamTransportOptions::default(), + }) + .await; + + assert!(matches!( + result, + Err(Error::Unsupported( + "messages event streaming provider registration" + )) + )); + } +} diff --git a/litellm-rust/crates/core/src/messages/transformation.rs b/litellm-rust/crates/core/src/messages/transformation.rs index 673a5728aca..1c3913a6802 100644 --- a/litellm-rust/crates/core/src/messages/transformation.rs +++ b/litellm-rust/crates/core/src/messages/transformation.rs @@ -1,5 +1,19 @@ -use super::types::{AnthropicMessagesRequest, AnthropicMessagesResponse}; use crate::Error; +use crate::streaming::StreamProvider; + +use super::types::{ + AnthropicMessagesRequest, AnthropicMessagesResponse, MessagesStreamEvent, MessagesStreamRequest, +}; + +pub trait MessagesStreamProvider: + StreamProvider +{ +} + +impl MessagesStreamProvider for T where + T: StreamProvider +{ +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MessagesAuthStrategy { diff --git a/litellm-rust/crates/core/src/messages/types.rs b/litellm-rust/crates/core/src/messages/types.rs index b9f807c29fd..94f979bb730 100644 --- a/litellm-rust/crates/core/src/messages/types.rs +++ b/litellm-rust/crates/core/src/messages/types.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use super::transformation::AnthropicMessagesProviderConfig; +use crate::streaming::{JsonObject, StreamTarget, StreamTransportOptions}; pub struct MessagesRequest<'a> { pub model: &'a str, @@ -47,6 +48,76 @@ pub struct ContentBlock { pub extra: Map, } +pub struct MessagesStreamRequest { + pub body: AnthropicMessagesRequest, + pub target: StreamTarget, + pub transport: StreamTransportOptions, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum MessagesStreamEvent { + MessageStart { + message: AnthropicMessagesResponse, + }, + ContentBlockStart { + index: u64, + content_block: JsonObject, + }, + ContentBlockDelta { + index: u64, + delta: JsonObject, + }, + ContentBlockStop { + index: u64, + }, + MessageDelta { + delta: JsonObject, + #[serde(default, skip_serializing_if = "Option::is_none")] + usage: Option, + }, + MessageStop, + Ping, + Error { + error: JsonObject, + }, +} + +#[cfg(test)] +mod stream_contract_tests { + use super::*; + + #[test] + fn message_stop_serializes_as_anthropic_event() { + assert_eq!( + serde_json::to_value(MessagesStreamEvent::MessageStop).expect("serializable event"), + serde_json::json!({"type": "message_stop"}) + ); + } + + #[test] + fn content_delta_keeps_typed_event_fields() { + let event = MessagesStreamEvent::ContentBlockDelta { + index: 0, + delta: JsonObject( + serde_json::json!({"type": "text_delta", "text": "hello"}) + .as_object() + .expect("object") + .clone(), + ), + }; + + assert_eq!( + serde_json::to_value(event).expect("serializable event"), + serde_json::json!({ + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "hello"} + }) + ); + } +} + #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct CacheControl { #[serde(rename = "type", skip_serializing_if = "Option::is_none")] diff --git a/litellm-rust/crates/core/src/responses/mod.rs b/litellm-rust/crates/core/src/responses/mod.rs index 5ec5a2caef8..7b211ba191b 100644 --- a/litellm-rust/crates/core/src/responses/mod.rs +++ b/litellm-rust/crates/core/src/responses/mod.rs @@ -1,3 +1,81 @@ pub mod instrumentation; pub mod types; pub mod websocket; + +use crate::error::Error; +use crate::streaming::OpenedStream; +use types::{ResponsesStreamEvent, ResponsesStreamRequest, ResponsesWebSocketRequest}; +use websocket::TypedResponsesWebSocketSession; + +pub async fn responses_stream( + _request: ResponsesStreamRequest, +) -> Result, Error> { + Err(Error::Unsupported( + "responses HTTP streaming provider registration", + )) +} + +pub async fn responses_websocket( + _request: ResponsesWebSocketRequest, +) -> Result, Error> { + Err(Error::Unsupported( + "responses WebSocket streaming provider registration", + )) +} + +#[cfg(test)] +mod stream_entrypoint_tests { + use serde_json::json; + + use super::*; + use crate::streaming::{ + ProviderCredentials, StreamProviderId, StreamTarget, StreamTransportOptions, + }; + + fn target() -> StreamTarget { + StreamTarget::new( + StreamProviderId::OpenAi, + ProviderCredentials::default(), + None, + ) + } + + #[tokio::test] + async fn typed_http_stream_declines_until_a_provider_is_registered() { + let body = serde_json::from_value(json!({ + "model": "gpt-5", + "input": "hello", + "stream": true + })) + .expect("valid Responses stream request"); + let result = responses_stream(ResponsesStreamRequest { + body, + target: target(), + transport: StreamTransportOptions::default(), + }) + .await; + + assert!(matches!( + result, + Err(Error::Unsupported( + "responses HTTP streaming provider registration" + )) + )); + } + + #[tokio::test] + async fn typed_websocket_declines_until_a_provider_is_registered() { + let result = responses_websocket(ResponsesWebSocketRequest { + target: target(), + transport: StreamTransportOptions::default(), + }) + .await; + + assert!(matches!( + result, + Err(Error::Unsupported( + "responses WebSocket streaming provider registration" + )) + )); + } +} diff --git a/litellm-rust/crates/core/src/responses/types.rs b/litellm-rust/crates/core/src/responses/types.rs index 4942309992e..c05408c2aa7 100644 --- a/litellm-rust/crates/core/src/responses/types.rs +++ b/litellm-rust/crates/core/src/responses/types.rs @@ -1,13 +1,46 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::{Map, Value}; +use crate::streaming::{JsonObject, StreamTarget, StreamTransportOptions}; + #[derive(Clone, Debug, PartialEq, Eq)] pub enum ResponsesWsEventType { ResponseCreate, ResponseCreated, + ResponseInProgress, + ResponseReasoningSummaryPartAdded, + ResponseReasoningSummaryTextDelta, + ResponseReasoningSummaryTextDone, + ResponseReasoningSummaryPartDone, + ResponseOutputItemAdded, + ResponseOutputTextDelta, + ResponseOutputTextAnnotationAdded, + ResponseOutputTextDone, + ResponseRefusalDelta, + ResponseRefusalDone, + ResponseFunctionCallArgumentsDelta, + ResponseFunctionCallArgumentsDone, + ResponseFileSearchCallInProgress, + ResponseFileSearchCallSearching, + ResponseFileSearchCallCompleted, + ResponseWebSearchCallInProgress, + ResponseWebSearchCallSearching, + ResponseWebSearchCallCompleted, + ResponseMcpListToolsInProgress, + ResponseMcpListToolsCompleted, + ResponseMcpListToolsFailed, + ResponseMcpCallInProgress, + ResponseMcpCallArgumentsDelta, + ResponseMcpCallArgumentsDone, + ResponseMcpCallCompleted, + ResponseMcpCallFailed, + ResponseContentPartAdded, + ResponseContentPartDone, + ResponseOutputItemDone, ResponseCompleted, ResponseFailed, ResponseIncomplete, + ImageGenerationPartialImage, Error, Other(String), } @@ -17,9 +50,40 @@ impl ResponsesWsEventType { match self { Self::ResponseCreate => "response.create", Self::ResponseCreated => "response.created", + Self::ResponseInProgress => "response.in_progress", + Self::ResponseReasoningSummaryPartAdded => "response.reasoning_summary_part.added", + Self::ResponseReasoningSummaryTextDelta => "response.reasoning_summary_text.delta", + Self::ResponseReasoningSummaryTextDone => "response.reasoning_summary_text.done", + Self::ResponseReasoningSummaryPartDone => "response.reasoning_summary_part.done", + Self::ResponseOutputItemAdded => "response.output_item.added", + Self::ResponseOutputTextDelta => "response.output_text.delta", + Self::ResponseOutputTextAnnotationAdded => "response.output_text.annotation.added", + Self::ResponseOutputTextDone => "response.output_text.done", + Self::ResponseRefusalDelta => "response.refusal.delta", + Self::ResponseRefusalDone => "response.refusal.done", + Self::ResponseFunctionCallArgumentsDelta => "response.function_call_arguments.delta", + Self::ResponseFunctionCallArgumentsDone => "response.function_call_arguments.done", + Self::ResponseFileSearchCallInProgress => "response.file_search_call.in_progress", + Self::ResponseFileSearchCallSearching => "response.file_search_call.searching", + Self::ResponseFileSearchCallCompleted => "response.file_search_call.completed", + Self::ResponseWebSearchCallInProgress => "response.web_search_call.in_progress", + Self::ResponseWebSearchCallSearching => "response.web_search_call.searching", + Self::ResponseWebSearchCallCompleted => "response.web_search_call.completed", + Self::ResponseMcpListToolsInProgress => "response.mcp_list_tools.in_progress", + Self::ResponseMcpListToolsCompleted => "response.mcp_list_tools.completed", + Self::ResponseMcpListToolsFailed => "response.mcp_list_tools.failed", + Self::ResponseMcpCallInProgress => "response.mcp_call.in_progress", + Self::ResponseMcpCallArgumentsDelta => "response.mcp_call_arguments.delta", + Self::ResponseMcpCallArgumentsDone => "response.mcp_call_arguments.done", + Self::ResponseMcpCallCompleted => "response.mcp_call.completed", + Self::ResponseMcpCallFailed => "response.mcp_call.failed", + Self::ResponseContentPartAdded => "response.content_part.added", + Self::ResponseContentPartDone => "response.content_part.done", + Self::ResponseOutputItemDone => "response.output_item.done", Self::ResponseCompleted => "response.completed", Self::ResponseFailed => "response.failed", Self::ResponseIncomplete => "response.incomplete", + Self::ImageGenerationPartialImage => "image_generation.partial_image", Self::Error => "error", Self::Other(value) => value, } @@ -44,9 +108,40 @@ impl<'de> Deserialize<'de> for ResponsesWsEventType { Ok(match value.as_str() { "response.create" => Self::ResponseCreate, "response.created" => Self::ResponseCreated, + "response.in_progress" => Self::ResponseInProgress, + "response.reasoning_summary_part.added" => Self::ResponseReasoningSummaryPartAdded, + "response.reasoning_summary_text.delta" => Self::ResponseReasoningSummaryTextDelta, + "response.reasoning_summary_text.done" => Self::ResponseReasoningSummaryTextDone, + "response.reasoning_summary_part.done" => Self::ResponseReasoningSummaryPartDone, + "response.output_item.added" => Self::ResponseOutputItemAdded, + "response.output_text.delta" => Self::ResponseOutputTextDelta, + "response.output_text.annotation.added" => Self::ResponseOutputTextAnnotationAdded, + "response.output_text.done" => Self::ResponseOutputTextDone, + "response.refusal.delta" => Self::ResponseRefusalDelta, + "response.refusal.done" => Self::ResponseRefusalDone, + "response.function_call_arguments.delta" => Self::ResponseFunctionCallArgumentsDelta, + "response.function_call_arguments.done" => Self::ResponseFunctionCallArgumentsDone, + "response.file_search_call.in_progress" => Self::ResponseFileSearchCallInProgress, + "response.file_search_call.searching" => Self::ResponseFileSearchCallSearching, + "response.file_search_call.completed" => Self::ResponseFileSearchCallCompleted, + "response.web_search_call.in_progress" => Self::ResponseWebSearchCallInProgress, + "response.web_search_call.searching" => Self::ResponseWebSearchCallSearching, + "response.web_search_call.completed" => Self::ResponseWebSearchCallCompleted, + "response.mcp_list_tools.in_progress" => Self::ResponseMcpListToolsInProgress, + "response.mcp_list_tools.completed" => Self::ResponseMcpListToolsCompleted, + "response.mcp_list_tools.failed" => Self::ResponseMcpListToolsFailed, + "response.mcp_call.in_progress" => Self::ResponseMcpCallInProgress, + "response.mcp_call_arguments.delta" => Self::ResponseMcpCallArgumentsDelta, + "response.mcp_call_arguments.done" => Self::ResponseMcpCallArgumentsDone, + "response.mcp_call.completed" => Self::ResponseMcpCallCompleted, + "response.mcp_call.failed" => Self::ResponseMcpCallFailed, + "response.content_part.added" => Self::ResponseContentPartAdded, + "response.content_part.done" => Self::ResponseContentPartDone, + "response.output_item.done" => Self::ResponseOutputItemDone, "response.completed" => Self::ResponseCompleted, "response.failed" => Self::ResponseFailed, "response.incomplete" => Self::ResponseIncomplete, + "image_generation.partial_image" => Self::ImageGenerationPartialImage, "error" => Self::Error, _ => Self::Other(value), }) @@ -84,6 +179,60 @@ pub struct ResponsesWsTransformResult { pub events: Vec, } +pub type ResponsesStreamEvent = ResponsesWsEvent; +pub type ResponseCommand = ResponsesWsEvent; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ResponsesInput { + Text(String), + Items(Vec), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ResponsesToolChoice { + Name(String), + Definition(JsonObject), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ResponsesStreamRequestBody { + pub model: String, + pub input: ResponsesInput, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub instructions: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub previous_response_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub store: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tools: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub include: Option>, + #[serde(flatten)] + pub extra: Map, +} + +pub struct ResponsesStreamRequest { + pub body: ResponsesStreamRequestBody, + pub target: StreamTarget, + pub transport: StreamTransportOptions, +} + +pub struct ResponsesWebSocketRequest { + pub target: StreamTarget, + pub transport: StreamTransportOptions, +} + impl ResponsesWsTransformResult { pub fn passthrough(event: ResponsesWsEvent) -> Self { Self { @@ -127,11 +276,14 @@ mod tests { let known: ResponsesWsEventType = serde_json::from_str("\"response.completed\"").expect("valid event type"); assert_eq!(known, ResponsesWsEventType::ResponseCompleted); - let unknown: ResponsesWsEventType = + let output_delta: ResponsesWsEventType = serde_json::from_str("\"response.output_text.delta\"").expect("valid event type"); + assert_eq!(output_delta, ResponsesWsEventType::ResponseOutputTextDelta); + let unknown: ResponsesWsEventType = + serde_json::from_str("\"response.future_event\"").expect("valid event type"); assert_eq!( unknown, - ResponsesWsEventType::Other("response.output_text.delta".to_string()) + ResponsesWsEventType::Other("response.future_event".to_string()) ); } @@ -163,4 +315,42 @@ mod tests { assert_eq!(flat.model(), Some("gpt-5")); assert_eq!(nested.model(), Some("gpt-5-mini")); } + + #[test] + fn stream_request_deserializes_the_public_responses_shape() { + let request: ResponsesStreamRequestBody = serde_json::from_value(serde_json::json!({ + "model": "gpt-5", + "input": "hello", + "stream": true, + "max_output_tokens": 32 + })) + .expect("public request shape"); + + assert_eq!(request.model, "gpt-5"); + assert_eq!(request.stream, Some(true)); + assert!(matches!(request.input, ResponsesInput::Text(ref text) if text == "hello")); + } + + #[test] + fn unknown_stream_events_round_trip_for_forward_compatibility() { + let event: ResponsesStreamEvent = serde_json::from_value(serde_json::json!({ + "type": "response.future_event", + "sequence_number": 7, + "future_field": "value" + })) + .expect("unknown event"); + + assert_eq!( + event.event_type, + ResponsesWsEventType::Other("response.future_event".to_string()) + ); + assert_eq!( + serde_json::to_value(event).expect("serializable event"), + serde_json::json!({ + "type": "response.future_event", + "sequence_number": 7, + "future_field": "value" + }) + ); + } } diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/core/src/responses/websocket.rs index 5d037e9cf1b..2432e46e017 100644 --- a/litellm-rust/crates/core/src/responses/websocket.rs +++ b/litellm-rust/crates/core/src/responses/websocket.rs @@ -1,6 +1,18 @@ use crate::Error; use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}; -use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult}; +use crate::responses::types::{ + ResponseCommand, ResponsesStreamEvent, ResponsesWsEvent, ResponsesWsEventType, + ResponsesWsTransformResult, +}; +use futures_util::future::BoxFuture; + +pub trait TypedResponsesWebSocketSession: Send + Sync { + fn send(&self, command: ResponseCommand) -> BoxFuture<'_, Result<(), Error>>; + + fn recv(&self) -> BoxFuture<'_, Result, Error>>; + + fn close(&self) -> BoxFuture<'_, Result<(), Error>>; +} pub trait ResponsesWebSocketProviderConfig: Sync { fn supports_native_websocket(&self) -> bool { From 22c50c07591ac9283857b480dceb5935a0ef00d4 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 07:13:58 -0700 Subject: [PATCH 63/67] feat(python-bridge): expose typed native streams --- .../crates/python-bridge/src/errors.rs | 15 + litellm-rust/crates/python-bridge/src/lib.rs | 10 + .../crates/python-bridge/src/routes/mod.rs | 5 +- .../python-bridge/src/routes/receiver.rs | 206 +++++++++ .../python-bridge/src/routes/runtime.rs | 100 +++++ .../python-bridge/src/routes/streaming.rs | 422 ++++++++++++++++++ .../rust_bridge/native_route_wheel_test.py | 90 ++++ 7 files changed, 847 insertions(+), 1 deletion(-) create mode 100644 litellm-rust/crates/python-bridge/src/routes/receiver.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/streaming.rs diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index d3e47937666..3e9bf9a3bbf 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -9,6 +9,10 @@ pyo3::create_exception!( "The route declined before calling the provider, so the host may retry on its own path." ); +pub(crate) fn declined(error: impl std::fmt::Display) -> PyErr { + RustBridgeDeclined::new_err(error.to_string()) +} + pyo3::create_exception!( _native, RustUpstreamError, @@ -34,6 +38,12 @@ pub(crate) fn core_error_to_pyerr(err: Error) -> PyErr { pub(crate) fn fallback_route_error_to_pyerr(err: Error) -> PyErr { match err { Error::Unsupported(_) => RustBridgeDeclined::new_err(err.to_string()), + other => executed_route_error_to_pyerr(other), + } +} + +pub(crate) fn executed_route_error_to_pyerr(err: Error) -> PyErr { + match err { Error::Http { status, body } => { RustUpstreamError::new_err((status, format!("{status}: {body}"))) } @@ -93,6 +103,11 @@ mod tests { .expect("upstream error should carry status and message"); assert_eq!(args, (expected.0, expected.1.to_string())); } + + let midstream = executed_route_error_to_pyerr(Error::Unsupported( + "a stream cannot fall back after opening", + )); + assert!(midstream.is_instance_of::(py)); }); } } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 47df6b905a8..65ce75a7768 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -102,6 +102,16 @@ mod tests { "chat_completions_decline", "chat_completions", "achat_completions", + "ChatCompletionsEventStream", + "MessagesEventStream", + "ResponsesEventStream", + "ResponsesWebSocketSession", + "chat_completions_stream", + "achat_completions_stream", + "messages_stream", + "amessages_stream", + "responses_stream", + "aresponses_stream", "ResponsesWebSocketConnection", "gil_stats", ]; diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index 3590ea3a769..1e64ae23248 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -8,10 +8,13 @@ mod audio_transcription; mod chat_completions; mod messages; mod ocr; +mod receiver; +mod streaming; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { ocr::register(module)?; audio_transcription::register(module)?; messages::register(module)?; - chat_completions::register(module) + chat_completions::register(module)?; + streaming::register(module) } diff --git a/litellm-rust/crates/python-bridge/src/routes/receiver.rs b/litellm-rust/crates/python-bridge/src/routes/receiver.rs new file mode 100644 index 00000000000..a97a256cfa2 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/receiver.rs @@ -0,0 +1,206 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use futures_util::{Stream, StreamExt, pin_mut}; +use litellm_core::Error; +use tokio::sync::{Mutex, mpsc}; +use tokio::task::JoinHandle; + +const BRIDGE_CHANNEL_CAPACITY: usize = 1; + +struct ReceiverState { + receiver: Mutex>>, + reading: AtomicBool, + closed: AtomicBool, + producer: std::sync::Mutex>>, +} + +impl Drop for ReceiverState { + fn drop(&mut self) { + if let Ok(producer) = self.producer.get_mut() + && let Some(producer) = producer.take() + { + producer.abort(); + } + } +} + +#[derive(Clone)] +pub(super) struct BridgeReceiver { + state: Arc>, +} + +struct ReadGuard<'a>(&'a AtomicBool); + +impl Drop for ReadGuard<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } +} + +impl BridgeReceiver { + pub(super) fn from_stream(stream: S) -> Self + where + S: Stream> + Send + 'static, + { + let (sender, receiver) = mpsc::channel(BRIDGE_CHANNEL_CAPACITY); + let producer = pyo3_async_runtimes::tokio::get_runtime().spawn(async move { + pin_mut!(stream); + while let Some(item) = stream.next().await { + let terminal = item.is_err(); + if sender.send(item).await.is_err() || terminal { + return; + } + } + }); + Self { + state: Arc::new(ReceiverState { + receiver: Mutex::new(receiver), + reading: AtomicBool::new(false), + closed: AtomicBool::new(false), + producer: std::sync::Mutex::new(Some(producer)), + }), + } + } + + pub(super) async fn next(&self) -> Result, Error> { + if self.state.closed.load(Ordering::Acquire) { + return Ok(None); + } + if self.state.reading.swap(true, Ordering::AcqRel) { + return Err(Error::InvalidRequest( + "native stream does not support concurrent reads".to_string(), + )); + } + let _guard = ReadGuard(&self.state.reading); + match self.state.receiver.lock().await.recv().await { + Some(Ok(item)) => Ok(Some(item)), + Some(Err(error)) => Err(error), + None => Ok(None), + } + } + + pub(super) fn close(&self) { + if self.state.closed.swap(true, Ordering::AcqRel) { + return; + } + if let Ok(mut producer) = self.state.producer.lock() + && let Some(producer) = producer.take() + { + producer.abort(); + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use futures_util::stream; + + use super::*; + + struct DropFlag(Arc); + + impl Drop for DropFlag { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } + } + + #[tokio::test] + async fn receiver_preserves_items_and_terminal_error() { + let receiver = BridgeReceiver::from_stream(stream::iter([ + Ok(vec![1_u8]), + Err(Error::Network("broken".to_string())), + Ok(vec![2_u8]), + ])); + + assert_eq!(receiver.next().await.expect("first item"), Some(vec![1])); + assert!(matches!( + receiver.next().await, + Err(Error::Network(message)) if message == "broken" + )); + assert_eq!(receiver.next().await.expect("closed after error"), None); + } + + #[tokio::test] + async fn close_unblocks_a_pending_read() { + let receiver = BridgeReceiver::>::from_stream(stream::pending()); + let pending = { + let receiver = receiver.clone(); + tokio::spawn(async move { receiver.next().await }) + }; + tokio::time::sleep(Duration::from_millis(10)).await; + receiver.close(); + + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), pending) + .await + .expect("read should unblock") + .expect("task should finish") + .expect("close is clean"), + None + ); + } + + #[tokio::test] + async fn capacity_one_stops_the_producer_from_draining_the_source() { + let polls = Arc::new(AtomicUsize::new(0)); + let source_polls = polls.clone(); + let source = stream::poll_fn(move |_| { + let item = source_polls.fetch_add(1, Ordering::SeqCst); + std::task::Poll::Ready(Some(Ok(item))) + }); + let receiver = BridgeReceiver::from_stream(source); + + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!(polls.load(Ordering::SeqCst), 2); + assert_eq!(receiver.next().await.expect("first item"), Some(0)); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!(polls.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn concurrent_reads_are_rejected() { + let receiver = BridgeReceiver::>::from_stream(stream::pending()); + let pending = { + let receiver = receiver.clone(); + tokio::spawn(async move { receiver.next().await }) + }; + tokio::time::sleep(Duration::from_millis(10)).await; + + assert!(matches!( + receiver.next().await, + Err(Error::InvalidRequest(message)) + if message == "native stream does not support concurrent reads" + )); + receiver.close(); + pending + .await + .expect("pending read task") + .expect("clean close"); + } + + #[tokio::test] + async fn dropping_the_last_receiver_cancels_the_producer() { + let dropped = Arc::new(AtomicBool::new(false)); + let producer_dropped = dropped.clone(); + let source = stream::once(async move { + let _flag = DropFlag(producer_dropped); + std::future::pending::, Error>>().await + }); + let receiver = BridgeReceiver::from_stream(source); + tokio::time::sleep(Duration::from_millis(10)).await; + + drop(receiver); + tokio::time::timeout(Duration::from_secs(1), async { + while !dropped.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + }) + .await + .expect("producer future should be dropped"); + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/runtime.rs b/litellm-rust/crates/python-bridge/src/routes/runtime.rs index 87a0c3e0104..75cce13bd36 100644 --- a/litellm-rust/crates/python-bridge/src/routes/runtime.rs +++ b/litellm-rust/crates/python-bridge/src/routes/runtime.rs @@ -65,6 +65,71 @@ where }) } +struct AttachedConversion { + value: T, + convert: C, +} + +impl<'py, T, C> IntoPyObject<'py> for AttachedConversion +where + C: FnOnce(Python<'py>, T) -> PyResult>, +{ + type Target = PyAny; + type Output = Bound<'py, PyAny>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> PyResult { + std::panic::catch_unwind(AssertUnwindSafe(|| (self.convert)(py, self.value))) + .map_err(panic_to_pyerr)? + .map(|value| value.into_bound(py)) + } +} + +pub(super) fn run_sync_with( + py: Python<'_>, + future: F, + map_error: fn(Error) -> PyErr, + convert: C, +) -> PyResult> +where + T: Send + 'static, + F: Future> + Send + 'static, + C: FnOnce(Python<'_>, T) -> PyResult>, +{ + if Handle::try_current().is_ok() { + return Err(PyRuntimeError::new_err( + "synchronous native routes cannot run from a Tokio context; use the async route", + )); + } + + let result = release_gil(py, move || { + pyo3_async_runtimes::tokio::get_runtime().block_on(wait_for_sync_result(future)) + })?; + let result = map_core_result(result, map_error)?; + std::panic::catch_unwind(AssertUnwindSafe(|| convert(py, result))).map_err(panic_to_pyerr)? +} + +pub(super) fn run_async_with( + py: Python<'_>, + future: F, + map_error: fn(Error) -> PyErr, + convert: C, +) -> PyResult> +where + T: Send + 'static, + F: Future> + Send + 'static, + C: for<'py> FnOnce(Python<'py>, T) -> PyResult> + Send + 'static, +{ + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let result = catch_route_panic(future).await?; + let result = map_core_result(result, map_error)?; + Ok(AttachedConversion { + value: result, + convert, + }) + }) +} + fn map_core_result(result: Result, map_error: fn(Error) -> PyErr) -> PyResult { match result { Ok(value) => Ok(value), @@ -159,6 +224,13 @@ mod tests { ) } + #[pyfunction] + fn async_custom_mapping(py: Python<'_>) -> PyResult> { + run_async_with(py, async { Ok("mapped") }, runtime_error, |py, value| { + Ok(value.into_pyobject(py)?.unbind().into_any()) + }) + } + #[pyfunction] fn runtime_worker_count() -> usize { pyo3_async_runtimes::tokio::get_runtime() @@ -420,4 +492,32 @@ asyncio.run(exercise()) .expect("result delivery should leave Tokio workers responsive"); }); } + + #[test] + fn async_custom_mapping_runs_during_attached_result_delivery() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "runtime").expect("module should be created"); + module + .add_function(wrap_pyfunction!(async_custom_mapping, &module).expect("function")) + .expect("function should register"); + let locals = PyDict::new(py); + locals + .set_item("runtime", &module) + .expect("module should enter Python locals"); + let code = CString::new( + r#" +import asyncio + +async def exercise(): + assert await runtime.async_custom_mapping() == "mapped" + +asyncio.run(exercise()) +"#, + ) + .expect("Python source should not contain null bytes"); + py.run(&code, Some(&locals), Some(&locals)) + .expect("custom mapping should reach the Python awaiter"); + }); + } } diff --git a/litellm-rust/crates/python-bridge/src/routes/streaming.rs b/litellm-rust/crates/python-bridge/src/routes/streaming.rs new file mode 100644 index 00000000000..7b4edc50d4d --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/streaming.rs @@ -0,0 +1,422 @@ +use std::sync::Arc; + +use litellm_core::chat_completions::chat_completions_stream as run_chat_completions_stream; +use litellm_core::chat_completions::types::{ + ChatCompletionsStreamRequest, ChatCompletionsStreamRequestBody, ChatStreamEvent, +}; +use litellm_core::messages::messages_event_stream as run_messages_stream; +use litellm_core::messages::types::{ + AnthropicMessagesRequest, MessagesStreamEvent, MessagesStreamRequest, +}; +use litellm_core::responses::responses_stream as run_responses_stream; +use litellm_core::responses::responses_websocket as run_responses_websocket; +use litellm_core::responses::types::{ + ResponseCommand, ResponsesStreamEvent, ResponsesStreamRequest, ResponsesStreamRequestBody, + ResponsesWebSocketRequest, +}; +use litellm_core::responses::websocket::TypedResponsesWebSocketSession; +use litellm_core::streaming::{ + JsonObject, OpenedStream, ProviderCredentials, StreamMetadata, StreamProviderId, StreamTarget, + StreamTransportOptions, +}; +use litellm_python_interop::{from_py, to_py}; +use pyo3::exceptions::{PyStopAsyncIteration, PyStopIteration}; +use pyo3::prelude::*; +use pyo3::types::{PyModule, PyType}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +use crate::marshal::{marshal_headers, optional_timeout}; +use crate::routes::receiver::BridgeReceiver; +use crate::routes::runtime::{run_async, run_async_with, run_sync_with}; + +struct TypedEventReceiver { + metadata: StreamMetadata, + receiver: BridgeReceiver, +} + +impl TypedEventReceiver +where + E: Send + 'static, +{ + fn from_opened(opened: OpenedStream) -> Self { + Self { + metadata: opened.metadata, + receiver: BridgeReceiver::from_stream(opened.events), + } + } +} + +fn next_event( + py: Python<'_>, + receiver: BridgeReceiver, + stop_iteration: bool, +) -> PyResult> +where + E: Serialize + Send + 'static, +{ + run_sync_with( + py, + async move { receiver.next().await }, + crate::errors::executed_route_error_to_pyerr, + move |py, event| match event { + Some(event) => to_py(py, &event), + None if stop_iteration => Err(PyStopIteration::new_err(())), + None => Ok(py.None()), + }, + ) +} + +fn anext_event( + py: Python<'_>, + receiver: BridgeReceiver, + stop_iteration: bool, +) -> PyResult> +where + E: Serialize + Send + 'static, +{ + run_async_with( + py, + async move { receiver.next().await }, + crate::errors::executed_route_error_to_pyerr, + move |py, event| match event { + Some(event) => to_py(py, &event), + None if stop_iteration => Err(PyStopAsyncIteration::new_err(())), + None => Ok(py.None()), + }, + ) +} + +macro_rules! event_stream_class { + ($class:ident, $event:ty) => { + #[pyclass] + struct $class { + inner: TypedEventReceiver<$event>, + } + + impl From> for $class { + fn from(opened: OpenedStream<$event>) -> Self { + Self { + inner: TypedEventReceiver::from_opened(opened), + } + } + } + + #[pymethods] + impl $class { + #[getter] + fn metadata(&self, py: Python<'_>) -> PyResult> { + to_py(py, &self.inner.metadata) + } + + fn next_event(&self, py: Python<'_>) -> PyResult> { + next_event(py, self.inner.receiver.clone(), false) + } + + fn anext_event<'py>(&self, py: Python<'py>) -> PyResult> { + anext_event(py, self.inner.receiver.clone(), false) + } + + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } + + fn __next__(&self, py: Python<'_>) -> PyResult> { + next_event(py, self.inner.receiver.clone(), true) + } + + fn __aiter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } + + fn __anext__<'py>(&self, py: Python<'py>) -> PyResult> { + anext_event(py, self.inner.receiver.clone(), true) + } + + fn close(&self) { + self.inner.receiver.close(); + } + + fn aclose<'py>(&self, py: Python<'py>) -> PyResult> { + let receiver = self.inner.receiver.clone(); + run_async( + py, + async move { + receiver.close(); + Ok(()) + }, + crate::errors::executed_route_error_to_pyerr, + ) + } + } + }; +} + +event_stream_class!(ChatCompletionsEventStream, ChatStreamEvent); +event_stream_class!(MessagesEventStream, MessagesStreamEvent); +event_stream_class!(ResponsesEventStream, ResponsesStreamEvent); + +#[derive(Default, Deserialize)] +struct PythonProviderCredentials { + api_key: Option, + aws_access_key_id: Option, + aws_secret_access_key: Option, + aws_session_token: Option, +} + +struct PythonStreamTarget { + provider: String, + credentials: Option>, + api_base: Option, +} + +struct PythonStreamTransport { + extra_headers: Option>, + timeout_seconds: Option, +} + +fn parse_call( + py: Python<'_>, + request: Py, + target: PythonStreamTarget, + transport: PythonStreamTransport, +) -> PyResult<(B, StreamTarget, StreamTransportOptions)> +where + B: DeserializeOwned, +{ + (|| -> PyResult<(B, StreamTarget, StreamTransportOptions)> { + let body = from_py(request.bind(py))?; + let credentials = target + .credentials + .map(|value| from_py::(value.bind(py))) + .transpose()? + .unwrap_or_default(); + let provider = StreamProviderId::try_from(target.provider.as_str()) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; + let target = StreamTarget::new( + provider, + ProviderCredentials::new( + credentials.api_key, + credentials.aws_access_key_id, + credentials.aws_secret_access_key, + credentials.aws_session_token, + ), + target.api_base, + ); + let extra_headers = transport + .extra_headers + .map(|value| from_py(value.bind(py))) + .transpose()?; + let forwarded_headers = marshal_headers(extra_headers)? + .into_iter() + .map(|(name, value)| litellm_core::streaming::Header { name, value }) + .collect(); + let transport = StreamTransportOptions::new( + forwarded_headers, + optional_timeout(transport.timeout_seconds), + ); + Ok((body, target, transport)) + })() + .map_err(crate::errors::declined) +} + +macro_rules! stream_openers { + ( + sync = $sync_name:ident, + asynchronous = $async_name:ident, + body = $body:ty, + request = $request:ident, + open = $open:path, + stream = $stream:ident + ) => { + #[pyfunction] + #[pyo3(signature = (request, provider, credentials=None, api_base=None, extra_headers=None, timeout_seconds=None))] + #[allow(clippy::too_many_arguments)] + fn $sync_name( + py: Python<'_>, + request: Py, + provider: String, + credentials: Option>, + api_base: Option, + extra_headers: Option>, + timeout_seconds: Option, + ) -> PyResult> { + let (body, target, transport) = parse_call::<$body>( + py, + request, + PythonStreamTarget { + provider, + credentials, + api_base, + }, + PythonStreamTransport { + extra_headers, + timeout_seconds, + }, + )?; + run_sync_with( + py, + async move { $open($request { body, target, transport }).await }, + crate::errors::fallback_route_error_to_pyerr, + |py, opened| Ok(Py::new(py, $stream::from(opened))?.into_any()), + ) + } + + #[pyfunction] + #[pyo3(signature = (request, provider, credentials=None, api_base=None, extra_headers=None, timeout_seconds=None))] + #[allow(clippy::too_many_arguments)] + fn $async_name( + py: Python<'_>, + request: Py, + provider: String, + credentials: Option>, + api_base: Option, + extra_headers: Option>, + timeout_seconds: Option, + ) -> PyResult> { + let (body, target, transport) = parse_call::<$body>( + py, + request, + PythonStreamTarget { + provider, + credentials, + api_base, + }, + PythonStreamTransport { + extra_headers, + timeout_seconds, + }, + )?; + run_async_with( + py, + async move { $open($request { body, target, transport }).await }, + crate::errors::fallback_route_error_to_pyerr, + |py, opened| Ok(Py::new(py, $stream::from(opened))?.into_any()), + ) + } + }; +} + +stream_openers! { + sync = chat_completions_stream, + asynchronous = achat_completions_stream, + body = ChatCompletionsStreamRequestBody, + request = ChatCompletionsStreamRequest, + open = run_chat_completions_stream, + stream = ChatCompletionsEventStream +} + +stream_openers! { + sync = messages_stream, + asynchronous = amessages_stream, + body = AnthropicMessagesRequest, + request = MessagesStreamRequest, + open = run_messages_stream, + stream = MessagesEventStream +} + +stream_openers! { + sync = responses_stream, + asynchronous = aresponses_stream, + body = ResponsesStreamRequestBody, + request = ResponsesStreamRequest, + open = run_responses_stream, + stream = ResponsesEventStream +} + +#[pyclass] +struct ResponsesWebSocketSession { + session: Arc, +} + +#[pymethods] +impl ResponsesWebSocketSession { + #[classmethod] + #[pyo3(signature = (provider, credentials=None, api_base=None, extra_headers=None, timeout_seconds=None))] + #[allow(clippy::too_many_arguments)] + fn connect<'py>( + _cls: &Bound<'py, PyType>, + py: Python<'py>, + provider: String, + credentials: Option>, + api_base: Option, + extra_headers: Option>, + timeout_seconds: Option, + ) -> PyResult> { + let empty_request = pyo3::types::PyDict::new(py).unbind().into_any(); + let (_, target, transport) = parse_call::( + py, + empty_request, + PythonStreamTarget { + provider, + credentials, + api_base, + }, + PythonStreamTransport { + extra_headers, + timeout_seconds, + }, + )?; + run_async_with( + py, + async move { + run_responses_websocket(ResponsesWebSocketRequest { target, transport }).await + }, + crate::errors::fallback_route_error_to_pyerr, + |py, session| { + Ok(Py::new( + py, + ResponsesWebSocketSession { + session: Arc::from(session), + }, + )? + .into_any()) + }, + ) + } + + fn send_event<'py>(&self, py: Python<'py>, command: Py) -> PyResult> { + let command: ResponseCommand = from_py(command.bind(py))?; + let session = self.session.clone(); + run_async( + py, + async move { session.send(command).await }, + crate::errors::executed_route_error_to_pyerr, + ) + } + + fn recv_event<'py>(&self, py: Python<'py>) -> PyResult> { + let session = self.session.clone(); + run_async_with( + py, + async move { session.recv().await }, + crate::errors::executed_route_error_to_pyerr, + |py, event| match event { + Some(event) => to_py(py, &event), + None => Ok(py.None()), + }, + ) + } + + fn close<'py>(&self, py: Python<'py>) -> PyResult> { + let session = self.session.clone(); + run_async( + py, + async move { session.close().await }, + crate::errors::executed_route_error_to_pyerr, + ) + } +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(chat_completions_stream, module)?)?; + module.add_function(wrap_pyfunction!(achat_completions_stream, module)?)?; + module.add_function(wrap_pyfunction!(messages_stream, module)?)?; + module.add_function(wrap_pyfunction!(amessages_stream, module)?)?; + module.add_function(wrap_pyfunction!(responses_stream, module)?)?; + module.add_function(wrap_pyfunction!(aresponses_stream, module)?) +} diff --git a/tests/test_litellm/rust_bridge/native_route_wheel_test.py b/tests/test_litellm/rust_bridge/native_route_wheel_test.py index 2789def7784..8f51c33565e 100644 --- a/tests/test_litellm/rust_bridge/native_route_wheel_test.py +++ b/tests/test_litellm/rust_bridge/native_route_wheel_test.py @@ -11,6 +11,7 @@ import sys import tempfile import threading import zipfile +from collections.abc import Awaitable, Callable from http.client import HTTPMessage from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -253,11 +254,100 @@ async def exercise_async_concurrency(native: object, api_base: str) -> None: assert_success("messages", response) +def streaming_route_kwargs(route: str) -> dict[str, object]: + if route == "chat_completions": + return { + "request": { + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hello"}], + "stream": True, + }, + "provider": "anthropic", + } + if route == "messages": + return { + "request": { + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 16, + "stream": True, + }, + "provider": "anthropic", + } + if route == "responses": + return { + "request": {"model": "gpt-5", "input": "hello", "stream": True}, + "provider": "openai", + } + raise AssertionError(f"unknown streaming route: {route}") + + +def assert_streaming_decline(native: object, operation: Callable[[], object]) -> None: + try: + operation() + except native.RustBridgeDeclined: + return + raise AssertionError("disabled native streaming route did not decline") + + +async def assert_async_streaming_decline( + native: object, + operation: Callable[[], Awaitable[object]], +) -> None: + try: + await operation() + except native.RustBridgeDeclined: + return + raise AssertionError("disabled async native streaming route did not decline") + + +def exercise_disabled_streaming_surface(native: object) -> None: + expected: Final = ( + "ChatCompletionsEventStream", + "MessagesEventStream", + "ResponsesEventStream", + "ResponsesWebSocketSession", + "chat_completions_stream", + "achat_completions_stream", + "messages_stream", + "amessages_stream", + "responses_stream", + "aresponses_stream", + ) + for name in expected: + if not hasattr(native, name): + raise AssertionError(f"packaged native bridge is missing {name}") + + for route in ("chat_completions", "messages", "responses"): + function: Final = getattr(native, f"{route}_stream") + kwargs: Final = streaming_route_kwargs(route) + assert_streaming_decline( + native, + lambda function=function, kwargs=kwargs: function(**kwargs), + ) + + async def exercise_async_surface() -> None: + for route in ("chat_completions", "messages", "responses"): + function: Final = getattr(native, f"a{route}_stream") + kwargs: Final = streaming_route_kwargs(route) + await assert_async_streaming_decline( + native, + lambda function=function, kwargs=kwargs: function(**kwargs), + ) + await assert_async_streaming_decline( + native, + lambda: native.ResponsesWebSocketSession.connect(provider="openai"), + ) + + asyncio.run(exercise_async_surface()) + + def exercise_routes(native_path: Path, api_base: str) -> object: native: Final = load_native(native_path) exercise_sync(native, api_base) asyncio.run(exercise_async(native, api_base)) asyncio.run(exercise_async_concurrency(native, api_base)) + exercise_disabled_streaming_surface(native) return native From 6b49cb963f80867fa82eeed0e435a32eef33563f Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 07:22:57 -0700 Subject: [PATCH 64/67] feat(python): adapt typed Rust streams --- litellm/llms/custom_httpx/llm_http_handler.py | 10 +- litellm/rust_bridge/responses_websocket.py | 61 +-- litellm/rust_bridge/streaming.py | 354 ++++++++++++++++++ .../custom_httpx/test_llm_http_handler.py | 10 +- .../responses/test_rust_bridge_websocket.py | 129 ++++++- .../rust_bridge/test_streaming.py | 327 ++++++++++++++++ 6 files changed, 851 insertions(+), 40 deletions(-) create mode 100644 litellm/rust_bridge/streaming.py create mode 100644 tests/test_litellm/rust_bridge/test_streaming.py diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a00e0fa0ae1..2002a1023f3 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -163,7 +163,11 @@ def _rust_responses_websocket_enabled( raw_request_override: Final = litellm_params.get("rust") request_override: Final = raw_request_override if isinstance(raw_request_override, bool) else None - return custom_llm_provider == "openai" and rust_enabled(request_override=request_override) + if custom_llm_provider != "openai" or not rust_enabled(request_override=request_override): + return False + from litellm.rust_bridge.streaming import supports_streaming + + return supports_streaming("responses", custom_llm_provider, "websocket") def _anthropic_messages_with_core_engine( @@ -6513,7 +6517,9 @@ class BaseLLMHTTPHandler: from litellm.rust_bridge import responses_websocket as rust_responses_websocket rust_execution: Final = await rust_responses_websocket.connect( - url=ws_url, + provider="openai", + api_key=api_key, + api_base=api_base, headers={str(key): str(value) for key, value in headers.items()}, timeout=timeout, ) diff --git a/litellm/rust_bridge/responses_websocket.py b/litellm/rust_bridge/responses_websocket.py index 4c1c2330f5c..54e8ee4bb2a 100644 --- a/litellm/rust_bridge/responses_websocket.py +++ b/litellm/rust_bridge/responses_websocket.py @@ -2,11 +2,16 @@ from __future__ import annotations +import json +from collections.abc import Mapping +from types import MappingProxyType from typing import Final, Protocol import httpx +from pydantic import TypeAdapter from websockets.exceptions import ConnectionClosedOK +from litellm.rust_bridge import streaming from litellm.rust_bridge.bindings import UNSET, NativeBinding, Unset from litellm.rust_bridge.runtime import ( BridgeErrorContext, @@ -19,11 +24,13 @@ from litellm.rust_bridge.runtime import ( ) from litellm.rust_bridge.timeouts import timeout_to_seconds +_EVENT_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + class RustResponsesWebSocket(Protocol): - async def send_text(self, text: str) -> None: ... + async def send_event(self, event: Mapping[str, object]) -> None: ... - async def recv_text(self) -> str | None: ... + async def recv_event(self) -> Mapping[str, object] | None: ... async def close(self) -> None: ... @@ -32,13 +39,15 @@ class RustResponsesWebSocketConnection(Protocol): @classmethod async def connect( cls, - url: str, - headers: dict[str, str], + provider: str, + credentials: Mapping[str, str] | None, + api_base: str | None, + extra_headers: Mapping[str, str] | None, timeout_seconds: float | None, ) -> RustResponsesWebSocket: ... -_CONNECTION: Final = NativeBinding[type[RustResponsesWebSocketConnection]]("ResponsesWebSocketConnection") +_CONNECTION: Final = NativeBinding[type[RustResponsesWebSocketConnection]]("ResponsesWebSocketSession") def set_rust_responses_websocket( @@ -53,52 +62,62 @@ def load_rust_responses_websocket() -> type[RustResponsesWebSocketConnection] | class _ConnectionAdapter: - def __init__(self, connection: RustResponsesWebSocket): + def __init__(self, connection: RustResponsesWebSocket, context: BridgeErrorContext): self._connection: Final = connection + self._context: Final = context self.core_engine: Final = CoreEngine.RUST async def send(self, text: str) -> None: + event: Final = _EVENT_ADAPTER.validate_json(text) await acall( - lambda: self._connection.send_text(text), - BridgeErrorContext(route="responses websocket", provider="openai", model=""), + lambda: self._connection.send_event(event), + self._context, ) async def recv(self) -> str: - message: Final = await acall( - self._connection.recv_text, - BridgeErrorContext(route="responses websocket", provider="openai", model=""), + event: Final = await acall( + self._connection.recv_event, + self._context, ) - if message is None: + if event is None: raise ConnectionClosedOK(None, None) - return message + return json.dumps(dict(event), separators=(",", ":")) # mutable-ok: JSON needs a concrete dict async def close(self) -> None: await acall( self._connection.close, - BridgeErrorContext(route="responses websocket", provider="openai", model=""), + self._context, ) async def connect( *, - url: str, - headers: dict[str, str], + provider: str, + api_key: str | None, + api_base: str | None, + headers: Mapping[str, str], timeout: float | httpx.Timeout | None, ) -> ExecutionResult[_ConnectionAdapter | None]: + context: Final = BridgeErrorContext(route="responses websocket", provider=provider, model="") + if not streaming.supports_streaming("responses", provider, "websocket"): + return ExecutionResult(value=None, source=CoreEngine.PYTHON) connection_type: Final = load_rust_responses_websocket() + credentials: Final = None if api_key is None else MappingProxyType({"api_key": api_key}) native_call: Final = ( None if connection_type is None else lambda: connection_type.connect( - url=url, - headers=headers, - timeout_seconds=timeout_to_seconds(timeout), + provider, + credentials, + api_base, + headers, + timeout_to_seconds(timeout), ) ) return await ainvoke( native_call=native_call, fallback=async_none, - adapt=_ConnectionAdapter, + adapt=lambda connection: _ConnectionAdapter(connection, context), mode=FallbackMode.PYTHON, - context=BridgeErrorContext(route="responses websocket", provider="openai", model=""), + context=context, ) diff --git a/litellm/rust_bridge/streaming.py b/litellm/rust_bridge/streaming.py new file mode 100644 index 00000000000..1b7fc1bb5e8 --- /dev/null +++ b/litellm/rust_bridge/streaming.py @@ -0,0 +1,354 @@ +from __future__ import annotations + +import json +from collections.abc import AsyncIterator, Iterator, Mapping +from functools import lru_cache +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable + +import httpx + +from litellm.rust_bridge.bindings import UNSET, NativeBinding, Unset +from litellm.rust_bridge.runtime import ( + BridgeErrorContext, + RustHandled, + aattempt, + acall, + attempt, + call, +) +from litellm.rust_bridge.timeouts import timeout_to_seconds + +if TYPE_CHECKING: + from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponsesAPIStreamingResponse + +StreamApi: TypeAlias = Literal["chat_completions", "messages", "responses"] +StreamTransport: TypeAlias = Literal["http", "websocket"] +Event: TypeAlias = Mapping[str, object] + + +@runtime_checkable +class RustEventStream(Protocol): + @property + def metadata(self) -> Mapping[str, object]: ... + + def next_event(self) -> Event | None: ... + + async def anext_event(self) -> Event | None: ... + + def close(self) -> None: ... + + async def aclose(self) -> None: ... + + +class RustStreamOpen(Protocol): + def __call__( + self, + request: Mapping[str, object], + provider: str, + credentials: Mapping[str, str] | None, + api_base: str | None, + extra_headers: Mapping[str, str] | None, + timeout_seconds: float | None, + ) -> object: ... + + +class RustAsyncStreamOpen(Protocol): + async def __call__( + self, + request: Mapping[str, object], + provider: str, + credentials: Mapping[str, str] | None, + api_base: str | None, + extra_headers: Mapping[str, str] | None, + timeout_seconds: float | None, + ) -> object: ... + + +class RustStreamCapability(Protocol): + def __call__(self, api: str, provider: str, transport: str) -> bool: ... + + +_CAPABILITY: Final = NativeBinding[RustStreamCapability]("supports_streaming") +_CHAT: Final = NativeBinding[RustStreamOpen]("chat_completions_stream") +_ACHAT: Final = NativeBinding[RustAsyncStreamOpen]("achat_completions_stream") +_MESSAGES: Final = NativeBinding[RustStreamOpen]("messages_stream") +_AMESSAGES: Final = NativeBinding[RustAsyncStreamOpen]("amessages_stream") +_RESPONSES: Final = NativeBinding[RustStreamOpen]("responses_stream") +_ARESPONSES: Final = NativeBinding[RustAsyncStreamOpen]("aresponses_stream") + + +def set_rust_streaming( + *, + capability: RustStreamCapability | None | Unset = UNSET, + chat: RustStreamOpen | None | Unset = UNSET, + achat: RustAsyncStreamOpen | None | Unset = UNSET, + messages: RustStreamOpen | None | Unset = UNSET, + amessages: RustAsyncStreamOpen | None | Unset = UNSET, + responses: RustStreamOpen | None | Unset = UNSET, + aresponses: RustAsyncStreamOpen | None | Unset = UNSET, +) -> None: + """Override native stream bindings in tests; passing ``None`` restores discovery.""" + + _CAPABILITY.update(capability) + _CHAT.update(chat) + _ACHAT.update(achat) + _MESSAGES.update(messages) + _AMESSAGES.update(amessages) + _RESPONSES.update(responses) + _ARESPONSES.update(aresponses) + + +def supports_streaming(api: StreamApi, provider: str, transport: StreamTransport = "http") -> bool: + capability: Final = _CAPABILITY.load() + if capability is None: + return False + try: + return capability(api, provider, transport) is True + except Exception: # noqa: BLE001 # extension-defined errors cannot be named statically + return False + + +def _sync_opener(api: StreamApi) -> RustStreamOpen | None: + match api: + case "chat_completions": + return _CHAT.load() + case "messages": + return _MESSAGES.load() + case "responses": + return _RESPONSES.load() + + +def _async_opener(api: StreamApi) -> RustAsyncStreamOpen | None: + match api: + case "chat_completions": + return _ACHAT.load() + case "messages": + return _AMESSAGES.load() + case "responses": + return _ARESPONSES.load() + + +def _context(api: StreamApi, provider: str, request: Mapping[str, object]) -> BridgeErrorContext: + model: Final = request.get("model") + return BridgeErrorContext( + route=f"{api.replace('_', ' ')} stream", + provider=provider, + model=model if isinstance(model, str) else "", + ) + + +def _adapt_stream( + stream: object, + *, + context: BridgeErrorContext, +) -> TypedEventStreamAdapter: + if not isinstance(stream, RustEventStream): + raise TypeError("native stream opener returned an invalid stream") + return TypedEventStreamAdapter(stream, context) + + +class TypedEventStreamAdapter: + def __init__(self, stream: RustEventStream, context: BridgeErrorContext) -> None: + self._stream: Final = stream + self._context: Final = context + self.metadata: Final = stream.metadata + self._mode: Literal["sync", "async"] | None = None + + def _claim(self, mode: Literal["sync", "async"]) -> None: + if self._mode is None: + self._mode = mode + return + if self._mode != mode: + raise RuntimeError("native stream cannot mix synchronous and asynchronous consumption") + + def __iter__(self) -> Iterator[Event]: + self._claim("sync") + return self + + def __next__(self) -> Event: + self._claim("sync") + event: Final = call(self._stream.next_event, self._context) + if event is None: + raise StopIteration + return event + + def __aiter__(self) -> AsyncIterator[Event]: + self._claim("async") + return self + + async def __anext__(self) -> Event: + self._claim("async") + event: Final = await acall(self._stream.anext_event, self._context) + if event is None: + raise StopAsyncIteration + return event + + def close(self) -> None: + call(self._stream.close, self._context) + + async def aclose(self) -> None: + await acall(self._stream.aclose, self._context) + + +class MessagesSseStreamAdapter: + def __init__(self, events: TypedEventStreamAdapter) -> None: + self._events: Final = events + self.metadata: Final = events.metadata + + def __iter__(self) -> Iterator[bytes]: + return (_event_to_sse(event) for event in self._events) + + async def __aiter__(self) -> AsyncIterator[bytes]: + async for event in self._events: + yield _event_to_sse(event) + + def close(self) -> None: + self._events.close() + + async def aclose(self) -> None: + await self._events.aclose() + + +class ResponsesSdkEventStreamAdapter: + def __init__(self, events: TypedEventStreamAdapter) -> None: + self._events: Final = events + self.metadata: Final = events.metadata + + def __iter__(self) -> Iterator[ResponsesAPIStreamingResponse]: + return (_responses_event_to_sdk(event) for event in self._events) + + async def __aiter__(self) -> AsyncIterator[ResponsesAPIStreamingResponse]: + async for event in self._events: + yield _responses_event_to_sdk(event) + + def close(self) -> None: + self._events.close() + + async def aclose(self) -> None: + await self._events.aclose() + + +def _responses_event_to_sdk(event: Event) -> ResponsesAPIStreamingResponse: + from litellm.types.llms.openai import GenericEvent + + event_type: Final = event.get("type") + model: Final = _responses_event_models().get(event_type) if isinstance(event_type, str) else None + return (model or GenericEvent).model_validate(event) + + +@lru_cache(maxsize=1) +def _responses_event_models() -> Mapping[str, type[BaseLiteLLMOpenAIResponseObject]]: + from litellm.types.llms import openai as openai_types + + return MappingProxyType( + { + "response.created": openai_types.ResponseCreatedEvent, + "response.in_progress": openai_types.ResponseInProgressEvent, + "response.completed": openai_types.ResponseCompletedEvent, + "response.failed": openai_types.ResponseFailedEvent, + "response.incomplete": openai_types.ResponseIncompleteEvent, + "response.reasoning_summary_part.added": openai_types.ResponsePartAddedEvent, + "response.reasoning_summary_text.delta": openai_types.ReasoningSummaryTextDeltaEvent, + "response.reasoning_summary_text.done": openai_types.ReasoningSummaryTextDoneEvent, + "response.reasoning_summary_part.done": openai_types.ReasoningSummaryPartDoneEvent, + "response.output_item.added": openai_types.OutputItemAddedEvent, + "response.output_item.done": openai_types.OutputItemDoneEvent, + "response.content_part.added": openai_types.ContentPartAddedEvent, + "response.content_part.done": openai_types.ContentPartDoneEvent, + "response.output_text.delta": openai_types.OutputTextDeltaEvent, + "response.output_text.annotation.added": openai_types.OutputTextAnnotationAddedEvent, + "response.output_text.done": openai_types.OutputTextDoneEvent, + "response.refusal.delta": openai_types.RefusalDeltaEvent, + "response.refusal.done": openai_types.RefusalDoneEvent, + "response.function_call_arguments.delta": openai_types.FunctionCallArgumentsDeltaEvent, + "response.function_call_arguments.done": openai_types.FunctionCallArgumentsDoneEvent, + "response.file_search_call.in_progress": openai_types.FileSearchCallInProgressEvent, + "response.file_search_call.searching": openai_types.FileSearchCallSearchingEvent, + "response.file_search_call.completed": openai_types.FileSearchCallCompletedEvent, + "response.web_search_call.in_progress": openai_types.WebSearchCallInProgressEvent, + "response.web_search_call.searching": openai_types.WebSearchCallSearchingEvent, + "response.web_search_call.completed": openai_types.WebSearchCallCompletedEvent, + "response.mcp_list_tools.in_progress": openai_types.MCPListToolsInProgressEvent, + "response.mcp_list_tools.completed": openai_types.MCPListToolsCompletedEvent, + "response.mcp_list_tools.failed": openai_types.MCPListToolsFailedEvent, + "response.mcp_call.in_progress": openai_types.MCPCallInProgressEvent, + "response.mcp_call_arguments.delta": openai_types.MCPCallArgumentsDeltaEvent, + "response.mcp_call_arguments.done": openai_types.MCPCallArgumentsDoneEvent, + "response.mcp_call.completed": openai_types.MCPCallCompletedEvent, + "response.mcp_call.failed": openai_types.MCPCallFailedEvent, + "image_generation.partial_image": openai_types.ImageGenerationPartialImageEvent, + "error": openai_types.ErrorEvent, + } + ) + + +def _event_to_sse(event: Event) -> bytes: + payload: Final = dict(event) # mutable-ok: JSON requires a concrete dict + return f"data: {json.dumps(payload, separators=(',', ':'))}\n\n".encode() + + +def open_stream( + *, + api: StreamApi, + provider: str, + request: Mapping[str, object], + credentials: Mapping[str, str] | None, + api_base: str | None, + extra_headers: Mapping[str, str] | None, + timeout: float | httpx.Timeout | None, +) -> TypedEventStreamAdapter | None: + if not supports_streaming(api, provider): + return None + opener: Final = _sync_opener(api) + context: Final = _context(api, provider, request) + result: Final = attempt( + native_call=( + None + if opener is None + else lambda: opener( + request, + provider, + credentials, + api_base, + extra_headers, + timeout_to_seconds(timeout), + ) + ), + adapt=lambda stream: _adapt_stream(stream, context=context), + context=context, + ) + return result.value if isinstance(result, RustHandled) else None + + +async def aopen_stream( + *, + api: StreamApi, + provider: str, + request: Mapping[str, object], + credentials: Mapping[str, str] | None, + api_base: str | None, + extra_headers: Mapping[str, str] | None, + timeout: float | httpx.Timeout | None, +) -> TypedEventStreamAdapter | None: + if not supports_streaming(api, provider): + return None + opener: Final = _async_opener(api) + context: Final = _context(api, provider, request) + result: Final = await aattempt( + native_call=( + None + if opener is None + else lambda: opener( + request, + provider, + credentials, + api_base, + extra_headers, + timeout_to_seconds(timeout), + ) + ), + adapt=lambda stream: _adapt_stream(stream, context=context), + context=context, + ) + return result.value if isinstance(result, RustHandled) else None diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 26f841c1146..e571cc4e263 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -2680,10 +2680,16 @@ async def test_generic_http_handler_async_streaming_forwards_provider_response_h (None, GenericLiteLLMParams(rust=True), False), ], ) -def test_the_rust_responses_websocket_needs_both_openai_and_the_rust_flag( +def test_the_rust_responses_websocket_needs_provider_flag_and_typed_capability( custom_llm_provider, litellm_params, expected ): - assert _rust_responses_websocket_enabled(custom_llm_provider, litellm_params) is expected + from litellm.rust_bridge import streaming + + streaming.set_rust_streaming(capability=lambda api, provider, transport: True) + try: + assert _rust_responses_websocket_enabled(custom_llm_provider, litellm_params) is expected + finally: + streaming.set_rust_streaming(capability=None) def test_a_plain_callback_does_not_advertise_a_pre_call_deployment_hook(monkeypatch): diff --git a/tests/test_litellm/responses/test_rust_bridge_websocket.py b/tests/test_litellm/responses/test_rust_bridge_websocket.py index 12f75bc32f0..744c5172689 100644 --- a/tests/test_litellm/responses/test_rust_bridge_websocket.py +++ b/tests/test_litellm/responses/test_rust_bridge_websocket.py @@ -1,29 +1,32 @@ from __future__ import annotations +from collections.abc import Mapping + import pytest +from litellm.exceptions import APIError from litellm.llms.custom_httpx.llm_http_handler import _rust_responses_websocket_enabled -from litellm.rust_bridge import bindings, configuration, responses_websocket +from litellm.rust_bridge import bindings, configuration, responses_websocket, streaming from litellm.types.router import GenericLiteLLMParams class _FakeNativeConnection: def __init__(self) -> None: - self.sent: list[str] = [] + self.sent: list[dict[str, object]] = [] self.closed = False - async def send_text(self, text: str) -> None: - self.sent.append(text) + async def send_event(self, event: Mapping[str, object]) -> None: + self.sent.append(dict(event)) - async def recv_text(self) -> str: - return "response.completed" + async def recv_event(self) -> Mapping[str, object]: + return {"type": "response.completed"} async def close(self) -> None: self.closed = True class _ClosedNativeConnection: - async def recv_text(self) -> None: + async def recv_event(self) -> None: return None @@ -31,19 +34,61 @@ class _FakeNativeBridge: @classmethod async def connect( cls, - *, - url: str, - headers: dict[str, str], + provider: str, + credentials: Mapping[str, str] | None, + api_base: str | None, + extra_headers: Mapping[str, str] | None, timeout_seconds: float | None, ) -> _FakeNativeConnection: return _FakeNativeConnection() +class _Declined(Exception): + pass + + +class _Upstream(Exception): + pass + + +class _NativeErrors: + RustBridgeDeclined = _Declined + RustUpstreamError = _Upstream + + +class _DecliningNativeBridge: + @classmethod + async def connect( + cls, + provider: str, + credentials: Mapping[str, str] | None, + api_base: str | None, + extra_headers: Mapping[str, str] | None, + timeout_seconds: float | None, + ) -> _FakeNativeConnection: + raise _Declined("provider unsupported") + + +class _FailingNativeBridge: + @classmethod + async def connect( + cls, + provider: str, + credentials: Mapping[str, str] | None, + api_base: str | None, + extra_headers: Mapping[str, str] | None, + timeout_seconds: float | None, + ) -> _FakeNativeConnection: + raise _Upstream(503, "request may have executed") + + @pytest.fixture(autouse=True) def reset_responses_websocket(): + streaming.set_rust_streaming(capability=None) responses_websocket.set_rust_responses_websocket(connection=None) configuration.reset_rust_configuration() yield + streaming.set_rust_streaming(capability=None) responses_websocket.set_rust_responses_websocket(connection=None) configuration.reset_rust_configuration() @@ -51,16 +96,26 @@ def reset_responses_websocket(): def test_rust_websocket_bridge_is_disabled_without_flag() -> None: assert not _rust_responses_websocket_enabled("openai", GenericLiteLLMParams()) assert not _rust_responses_websocket_enabled("anthropic", GenericLiteLLMParams(rust=True)) + assert not _rust_responses_websocket_enabled("openai", GenericLiteLLMParams(rust=True)) + + +def test_injected_typed_capability_enables_the_gate() -> None: + streaming.set_rust_streaming( + capability=lambda api, provider, transport: (api, provider, transport) == ("responses", "openai", "websocket") + ) + assert _rust_responses_websocket_enabled("openai", GenericLiteLLMParams(rust=True)) def test_explicit_false_overrides_process_enable() -> None: + streaming.set_rust_streaming(capability=lambda api, provider, transport: True) configuration.use_litellm_rust(True) assert not _rust_responses_websocket_enabled("openai", GenericLiteLLMParams(rust=False)) def test_process_enable_applies_without_request_override() -> None: + streaming.set_rust_streaming(capability=lambda api, provider, transport: True) configuration.use_litellm_rust(True) assert _rust_responses_websocket_enabled("openai", GenericLiteLLMParams()) @@ -68,7 +123,10 @@ def test_process_enable_applies_without_request_override() -> None: @pytest.mark.asyncio async def test_adapter_raises_clean_close_when_rust_connection_ends() -> None: - adapter = responses_websocket._ConnectionAdapter(_ClosedNativeConnection()) + adapter = responses_websocket._ConnectionAdapter( + _ClosedNativeConnection(), + responses_websocket.BridgeErrorContext(route="responses websocket", provider="openai", model=""), + ) with pytest.raises(responses_websocket.ConnectionClosedOK): await adapter.recv() @@ -77,9 +135,12 @@ async def test_adapter_raises_clean_close_when_rust_connection_ends() -> None: @pytest.mark.asyncio async def test_bridge_unavailable_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) + streaming.set_rust_streaming(capability=lambda api, provider, transport: True) result = await responses_websocket.connect( - url="wss://example.test/responses", + provider="openai", + api_key=None, + api_base="https://example.test", headers={}, timeout=None, ) @@ -91,10 +152,13 @@ async def test_bridge_unavailable_returns_none(monkeypatch: pytest.MonkeyPatch) async def test_enabled_bridge_connects_and_adapts_socket( monkeypatch: pytest.MonkeyPatch, ) -> None: + streaming.set_rust_streaming(capability=lambda api, provider, transport: True) responses_websocket.set_rust_responses_websocket(connection=_FakeNativeBridge) result = await responses_websocket.connect( - url="wss://example.test/responses", + provider="openai", + api_key="key", + api_base="https://example.test", headers={"Authorization": "Bearer key"}, timeout=1.0, ) @@ -103,6 +167,41 @@ async def test_enabled_bridge_connects_and_adapts_socket( connection = result.value assert connection is not None assert connection.core_engine is responses_websocket.CoreEngine.RUST - await connection.send("response.create") - assert await connection.recv() == "response.completed" + await connection.send('{"type":"response.create","model":"gpt-5"}') + assert await connection.recv() == '{"type":"response.completed"}' await connection.close() + + +@pytest.mark.asyncio +async def test_declined_connect_falls_back_before_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(bindings, "get_native_bridge", lambda: _NativeErrors()) + streaming.set_rust_streaming(capability=lambda api, provider, transport: True) + responses_websocket.set_rust_responses_websocket(connection=_DecliningNativeBridge) + + result = await responses_websocket.connect( + provider="openai", + api_key="key", + api_base="https://example.test", + headers={}, + timeout=None, + ) + + assert result.value is None + assert result.source is responses_websocket.CoreEngine.PYTHON + + +@pytest.mark.asyncio +async def test_upstream_connect_failure_never_falls_back(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(bindings, "get_native_bridge", lambda: _NativeErrors()) + streaming.set_rust_streaming(capability=lambda api, provider, transport: True) + responses_websocket.set_rust_responses_websocket(connection=_FailingNativeBridge) + + with pytest.raises(APIError, match="request may have executed") as caught: + await responses_websocket.connect( + provider="openai", + api_key="key", + api_base="https://example.test", + headers={}, + timeout=None, + ) + assert caught.value.headers["x-litellm-core"] == "rust" diff --git a/tests/test_litellm/rust_bridge/test_streaming.py b/tests/test_litellm/rust_bridge/test_streaming.py new file mode 100644 index 00000000000..585187fa5e9 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_streaming.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from typing import Final +from unittest.mock import MagicMock + +import pytest + +from litellm.exceptions import APIError +from litellm.rust_bridge import bindings, streaming +from litellm.rust_bridge.runtime import BridgeErrorContext + + +class _FakeEventStream: + def __init__(self, events: tuple[Mapping[str, object], ...]) -> None: + self.metadata: Final = { + "status_code": 200, + "provider": "anthropic", + "transport": "http", + "response_headers": [{"name": "x-test", "value": "ready"}], + } + self._events: Final = iter(events) + self.closed = False + + def next_event(self) -> Mapping[str, object] | None: + if self.closed: + return None + return next(self._events, None) + + async def anext_event(self) -> Mapping[str, object] | None: + return self.next_event() + + def close(self) -> None: + self.closed = True + + async def aclose(self) -> None: + self.close() + + +class _RecordingOpen: + def __init__(self, events: tuple[Mapping[str, object], ...]) -> None: + self._events: Final = events + self.calls = 0 + + def __call__( + self, + request: Mapping[str, object], + provider: str, + credentials: Mapping[str, str] | None, + api_base: str | None, + extra_headers: Mapping[str, str] | None, + timeout_seconds: float | None, + ) -> _FakeEventStream: + self.calls += 1 + return _FakeEventStream(self._events) + + +class _RecordingAsyncOpen: + def __init__(self, events: tuple[Mapping[str, object], ...]) -> None: + self._events: Final = events + self.calls = 0 + + async def __call__( + self, + request: Mapping[str, object], + provider: str, + credentials: Mapping[str, str] | None, + api_base: str | None, + extra_headers: Mapping[str, str] | None, + timeout_seconds: float | None, + ) -> _FakeEventStream: + self.calls += 1 + return _FakeEventStream(self._events) + + +class _Declined(Exception): + pass + + +class _Upstream(Exception): + pass + + +class _NativeErrors: + RustBridgeDeclined = _Declined + RustUpstreamError = _Upstream + + +class _FailingOpen: + def __init__(self, error: Exception) -> None: + self._error: Final = error + + def __call__( + self, + request: Mapping[str, object], + provider: str, + credentials: Mapping[str, str] | None, + api_base: str | None, + extra_headers: Mapping[str, str] | None, + timeout_seconds: float | None, + ) -> _FakeEventStream: + raise self._error + + +class _FailingEventStream(_FakeEventStream): + def next_event(self) -> Mapping[str, object] | None: + raise _Upstream(502, "stream interrupted") + + +@pytest.fixture(autouse=True) +def reset_bridge() -> Iterator[None]: + streaming.set_rust_streaming( + capability=None, + chat=None, + achat=None, + messages=None, + amessages=None, + responses=None, + aresponses=None, + ) + yield + streaming.set_rust_streaming( + capability=None, + chat=None, + achat=None, + messages=None, + amessages=None, + responses=None, + aresponses=None, + ) + + +def _chat_event(text: str) -> Mapping[str, object]: + return { + "text": text, + "tool_use": None, + "is_finished": False, + "finish_reason": "", + "usage": None, + } + + +def _context() -> BridgeErrorContext: + return BridgeErrorContext(route="chat completions stream", provider="anthropic", model="claude") + + +def test_no_native_capability_keeps_every_provider_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + opener: Final = _RecordingOpen((_chat_event("unused"),)) + monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) + streaming.set_rust_streaming(chat=opener) + + result: Final = streaming.open_stream( + api="chat_completions", + provider="anthropic", + request={"model": "claude", "messages": []}, + credentials={"api_key": "test"}, + api_base=None, + extra_headers=None, + timeout=None, + ) + + assert result is None + assert opener.calls == 0 + + +def test_disabled_capability_never_calls_native() -> None: + opener: Final = _RecordingOpen((_chat_event("unused"),)) + streaming.set_rust_streaming(capability=lambda api, provider, transport: False, chat=opener) + + result: Final = streaming.open_stream( + api="chat_completions", + provider="anthropic", + request={"model": "claude", "messages": []}, + credentials=None, + api_base=None, + extra_headers=None, + timeout=None, + ) + + assert result is None + assert opener.calls == 0 + + +def test_declined_open_failure_falls_back(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(bindings, "get_native_bridge", lambda: _NativeErrors()) + streaming.set_rust_streaming( + capability=lambda api, provider, transport: True, + chat=_FailingOpen(_Declined("unsupported request")), + ) + + result: Final = streaming.open_stream( + api="chat_completions", + provider="anthropic", + request={"model": "claude", "messages": []}, + credentials=None, + api_base=None, + extra_headers=None, + timeout=None, + ) + + assert result is None + + +def test_upstream_open_failure_never_falls_back(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(bindings, "get_native_bridge", lambda: _NativeErrors()) + streaming.set_rust_streaming( + capability=lambda api, provider, transport: True, + chat=_FailingOpen(_Upstream(503, "connection closed after request")), + ) + + with pytest.raises(APIError, match="connection closed after request") as caught: + streaming.open_stream( + api="chat_completions", + provider="anthropic", + request={"model": "claude", "messages": []}, + credentials=None, + api_base=None, + extra_headers=None, + timeout=None, + ) + assert caught.value.headers["x-litellm-core"] == "rust" + + +def test_midstream_failure_preserves_rust_provenance(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(bindings, "get_native_bridge", lambda: _NativeErrors()) + events: Final = streaming.TypedEventStreamAdapter(_FailingEventStream(()), _context()) + + with pytest.raises(APIError, match="stream interrupted") as caught: + next(events) + assert caught.value.headers["x-litellm-core"] == "rust" + + +def test_sync_typed_events_preserve_shape_metadata_and_close() -> None: + opener: Final = _RecordingOpen((_chat_event("one"), _chat_event("two"))) + streaming.set_rust_streaming(capability=lambda api, provider, transport: True, chat=opener) + result: Final = streaming.open_stream( + api="chat_completions", + provider="anthropic", + request={"model": "claude", "messages": []}, + credentials=None, + api_base=None, + extra_headers=None, + timeout=1.0, + ) + + assert result is not None + assert tuple(event["text"] for event in result) == ("one", "two") + assert result.metadata["provider"] == "anthropic" + result.close() + assert opener.calls == 1 + + +def test_chat_events_flow_through_custom_stream_wrapper() -> None: + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.types.utils import ModelResponseStream + + native: Final = _FakeEventStream((_chat_event("hello"),)) + events: Final = streaming.TypedEventStreamAdapter(native, _context()) + wrapper: Final = CustomStreamWrapper( + completion_stream=events, + model="claude", + logging_obj=MagicMock(), + custom_llm_provider="anthropic", + ) + + chunk: Final = next(wrapper) + assert isinstance(chunk, ModelResponseStream) + assert chunk.choices[0].delta.content == "hello" + + +@pytest.mark.asyncio +async def test_async_typed_events_and_cancellation() -> None: + opener: Final = _RecordingAsyncOpen((_chat_event("one"), _chat_event("two"))) + streaming.set_rust_streaming(capability=lambda api, provider, transport: True, achat=opener) + result: Final = await streaming.aopen_stream( + api="chat_completions", + provider="anthropic", + request={"model": "claude", "messages": []}, + credentials=None, + api_base=None, + extra_headers=None, + timeout=None, + ) + + assert result is not None + assert tuple(event["text"] for event in [event async for event in result]) == ("one", "two") + await result.aclose() + + +def test_messages_events_are_wrapped_in_existing_sse_bytes() -> None: + events: Final = streaming.TypedEventStreamAdapter(_FakeEventStream(({"type": "message_stop"},)), _context()) + messages: Final = streaming.MessagesSseStreamAdapter(events) + + assert tuple(messages) == (b'data: {"type":"message_stop"}\n\n',) + + +def test_responses_events_are_validated_into_existing_sdk_objects() -> None: + from litellm.types.llms.openai import OutputTextDeltaEvent + + native: Final = _FakeEventStream( + ( + { + "type": "response.output_text.delta", + "item_id": "item_1", + "output_index": 0, + "content_index": 0, + "delta": "hello", + }, + ) + ) + responses: Final = streaming.ResponsesSdkEventStreamAdapter(streaming.TypedEventStreamAdapter(native, _context())) + + event: Final = next(iter(responses)) + assert isinstance(event, OutputTextDeltaEvent) + assert event.delta == "hello" + + +@pytest.mark.asyncio +async def test_rejects_mixed_sync_and_async_consumption() -> None: + events: Final = streaming.TypedEventStreamAdapter(_FakeEventStream((_chat_event("one"),)), _context()) + assert tuple(events) == (_chat_event("one"),) + + with pytest.raises(RuntimeError, match="cannot mix"): + async for _ in events: + pass From a86333c0fbcd1de2d87a13aa210ccf1b9c1ae259 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 06:50:09 -0700 Subject: [PATCH 65/67] feat(rust): define canonical OCR contracts --- litellm-rust/Cargo.lock | 4 + litellm-rust/Cargo.toml | 3 + litellm-rust/crates/core/Cargo.toml | 3 + litellm-rust/crates/core/src/ocr/canonical.rs | 234 ++++++++++++++++++ litellm-rust/crates/core/src/ocr/mod.rs | 3 + litellm-rust/crates/core/src/ocr/policy.rs | 106 ++++++++ litellm-rust/crates/core/src/ocr/response.rs | 79 ++++++ litellm-rust/crates/core/src/ocr/types.rs | 28 +++ 8 files changed, 460 insertions(+) create mode 100644 litellm-rust/crates/core/src/ocr/canonical.rs create mode 100644 litellm-rust/crates/core/src/ocr/policy.rs create mode 100644 litellm-rust/crates/core/src/ocr/response.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index b2f788beb37..603047db7dc 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1428,7 +1428,9 @@ dependencies = [ "aws-sigv4", "aws-smithy-runtime-api", "aws-types", + "bytes", "futures-util", + "mime", "rand 0.8.7", "reqwest", "serde", @@ -1436,6 +1438,7 @@ dependencies = [ "sha2 0.10.9", "thiserror 2.0.19", "tokio", + "url", ] [[package]] @@ -2720,6 +2723,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index f99d7b47918..69728a652bd 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -33,6 +33,9 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } base64 = "0.22" +bytes = "1" +mime = "0.3" +url = { version = "2", features = ["serde"] } [profile.release] opt-level = 3 diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 2b7c76a9bec..46e22bc866a 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -7,11 +7,14 @@ repository.workspace = true [dependencies] futures-util.workspace = true +bytes.workspace = true +mime.workspace = true rand.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true +url.workspace = true sha2.workspace = true aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true } diff --git a/litellm-rust/crates/core/src/ocr/canonical.rs b/litellm-rust/crates/core/src/ocr/canonical.rs new file mode 100644 index 00000000000..8d77e5509a9 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/canonical.rs @@ -0,0 +1,234 @@ +use std::collections::BTreeMap; + +use bytes::Bytes; +use mime::Mime; +use serde_json::Value; +use thiserror::Error; +use url::Url; + +use super::policy::OcrCanonicalField; +use super::types::{Field, OcrDialectId}; + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum CanonicalOcrError { + #[error("invalid OCR parameter {field}: {reason}")] + InvalidParameter { + field: &'static str, + reason: &'static str, + }, + #[error("provider extra collides with canonical OCR field: {0}")] + ExtraCollidesWithCanonical(String), + #[error("LiteLLM control cannot enter OCR provider extras: {0}")] + ReservedExtra(String), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DocumentKind { + Image, + Pdf, +} + +#[derive(Clone, PartialEq)] +pub enum OcrDocument { + RemoteUrl { + kind: DocumentKind, + url: Url, + }, + Inline { + kind: DocumentKind, + media_type: Mime, + bytes: Bytes, + }, + ProviderReference { + provider: OcrDialectId, + id: String, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PageSelection(Vec); + +impl PageSelection { + pub fn new(pages: impl IntoIterator) -> Self { + Self(pages.into_iter().collect()) + } + + pub fn pages(&self) -> &[u32] { + &self.0 + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct AnnotationFormat(Value); + +impl AnnotationFormat { + pub fn new(schema: Value) -> Result { + if !schema.is_object() { + return Err(CanonicalOcrError::InvalidParameter { + field: "annotation_format", + reason: "must be a JSON object", + }); + } + Ok(Self(schema)) + } + + pub fn as_value(&self) -> &Value { + &self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TableFormat { + Html, + Markdown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConfidenceScoresGranularity { + Page, + Word, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct OcrOutputOptions { + pub include_image_base64: Field, + pub image_limit: Field, + pub image_min_size: Field, + pub bbox_annotation_format: Field, + pub document_annotation_format: Field, + pub document_annotation_prompt: Field, + pub extract_header: Field, + pub extract_footer: Field, + pub table_format: Field, + pub confidence_scores_granularity: Field, + pub include_blocks: Field, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OcrRequestId(String); + +impl OcrRequestId { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + if value.trim().is_empty() { + return Err(CanonicalOcrError::InvalidParameter { + field: "id", + reason: "must not be blank", + }); + } + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ExplicitProviderExtras { + dialect: OcrDialectId, + fields: BTreeMap, +} + +impl ExplicitProviderExtras { + pub fn try_new( + dialect: OcrDialectId, + fields: BTreeMap, + ) -> Result { + if let Some(field) = fields.keys().find(|field| { + OcrCanonicalField::from_wire_name(field).is_some() || is_litellm_control(field) + }) { + return Err(if OcrCanonicalField::from_wire_name(field).is_some() { + CanonicalOcrError::ExtraCollidesWithCanonical(field.clone()) + } else { + CanonicalOcrError::ReservedExtra(field.clone()) + }); + } + Ok(Self { dialect, fields }) + } + + pub fn dialect(&self) -> OcrDialectId { + self.dialect + } + + pub fn fields(&self) -> &BTreeMap { + &self.fields + } +} + +fn is_litellm_control(field: &str) -> bool { + field.starts_with("litellm_") + || matches!( + field, + "api_base" + | "api_key" + | "custom_llm_provider" + | "fallbacks" + | "metadata" + | "mock_response" + | "num_retries" + | "request_timeout" + | "retry_policy" + ) +} + +#[derive(Clone, PartialEq)] +pub struct CanonicalOcrRequest { + pub model: String, + pub document: OcrDocument, + pub pages: Field, + pub output: OcrOutputOptions, + pub request_id: Field, + pub provider_extras: ExplicitProviderExtras, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn extras_reject_canonical_field_collisions() { + let error = ExplicitProviderExtras::try_new( + OcrDialectId::Mistral, + BTreeMap::from([("pages".to_string(), json!([0]))]), + ) + .expect_err("canonical fields cannot enter extras"); + assert_eq!( + error, + CanonicalOcrError::ExtraCollidesWithCanonical("pages".to_string()) + ); + } + + #[test] + fn extras_reject_litellm_controls() { + for field in ["request_timeout", "litellm_future_control"] { + let error = ExplicitProviderExtras::try_new( + OcrDialectId::Mistral, + BTreeMap::from([(field.to_string(), json!(true))]), + ) + .expect_err("LiteLLM controls cannot enter extras"); + assert_eq!(error, CanonicalOcrError::ReservedExtra(field.to_string())); + } + } + + #[test] + fn extras_remain_bound_to_one_dialect() { + let extras = ExplicitProviderExtras::try_new( + OcrDialectId::ReductoV3, + BTreeMap::from([("chunking".to_string(), json!({"size": 1}))]), + ) + .expect("provider field is accepted"); + assert_eq!(extras.dialect(), OcrDialectId::ReductoV3); + assert_eq!(extras.fields()["chunking"], json!({"size": 1})); + } + + #[test] + fn canonical_fields_preserve_absent_null_and_value() { + assert_ne!(Field::::Absent, Field::Null); + assert_ne!( + Field::Null, + Field::Value(PageSelection::new([0_u32, 2_u32])) + ); + } +} diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index ec2fbb969a6..d28291a971e 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -1,2 +1,5 @@ +pub mod canonical; +pub mod policy; +pub mod response; pub mod transformation; pub mod types; diff --git a/litellm-rust/crates/core/src/ocr/policy.rs b/litellm-rust/crates/core/src/ocr/policy.rs new file mode 100644 index 00000000000..ae0d68157b6 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/policy.rs @@ -0,0 +1,106 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ParameterDisposition { + Forward, + Rename(&'static str), + Transform, + ConsumeAsConfiguration, + Reject, +} + +macro_rules! ocr_parameter_schema { + ($(($variant:ident, $field:ident, $wire_name:literal)),+ $(,)?) => { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum OcrCanonicalField { + $($variant),+ + } + + impl OcrCanonicalField { + pub const ALL: [Self; ocr_parameter_schema!(@count $($variant),+)] = [ + $(Self::$variant),+ + ]; + + pub const fn wire_name(self) -> &'static str { + match self { + $(Self::$variant => $wire_name),+ + } + } + + pub fn from_wire_name(value: &str) -> Option { + match value { + $($wire_name => Some(Self::$variant)),+, + _ => None, + } + } + } + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub struct OcrParameterPolicy { + $(pub $field: ParameterDisposition),+ + } + + impl OcrParameterPolicy { + pub const fn disposition(self, field: OcrCanonicalField) -> ParameterDisposition { + match field { + $(OcrCanonicalField::$variant => self.$field),+ + } + } + } + }; + (@count $($item:ident),+) => { + <[()]>::len(&[$(ocr_parameter_schema!(@replace $item ())),+]) + }; + (@replace $_item:ident $sub:expr) => { $sub }; +} + +ocr_parameter_schema!( + (Pages, pages, "pages"), + ( + IncludeImageBase64, + include_image_base64, + "include_image_base64" + ), + (ImageLimit, image_limit, "image_limit"), + (ImageMinSize, image_min_size, "image_min_size"), + ( + BboxAnnotationFormat, + bbox_annotation_format, + "bbox_annotation_format" + ), + ( + DocumentAnnotationFormat, + document_annotation_format, + "document_annotation_format" + ), + ( + DocumentAnnotationPrompt, + document_annotation_prompt, + "document_annotation_prompt" + ), + (ExtractHeader, extract_header, "extract_header"), + (ExtractFooter, extract_footer, "extract_footer"), + (TableFormat, table_format, "table_format"), + ( + ConfidenceScoresGranularity, + confidence_scores_granularity, + "confidence_scores_granularity" + ), + (IncludeBlocks, include_blocks, "include_blocks"), + (RequestId, request_id, "id"), +); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_canonical_field_round_trips_through_its_wire_name() { + assert_eq!(OcrCanonicalField::ALL.len(), 13); + for field in OcrCanonicalField::ALL { + assert_eq!( + OcrCanonicalField::from_wire_name(field.wire_name()), + Some(field) + ); + } + assert_eq!(OcrCanonicalField::from_wire_name("provider_private"), None); + } +} diff --git a/litellm-rust/crates/core/src/ocr/response.rs b/litellm-rust/crates/core/src/ocr/response.rs new file mode 100644 index 00000000000..1338d99ae2d --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/response.rs @@ -0,0 +1,79 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OcrPage { + pub index: u32, + pub markdown: String, +} + +#[derive(Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct OcrUsage { + pub pages_processed: Option, + pub credits: Option, + pub document_size_bytes: Option, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct NormalizedOcr { + pub pages: Vec, + pub model: String, + pub document_annotation: Option, + pub content: Option, + pub tables: Option>, + pub key_value_pairs: Option>, + pub usage: OcrUsage, +} + +#[derive(Clone, PartialEq)] +pub struct NativeOcrPayload(Value); + +impl NativeOcrPayload { + pub fn new(value: Value) -> Self { + Self(value) + } + + pub fn as_value(&self) -> &Value { + &self.0 + } + + pub fn into_value(self) -> Value { + self.0 + } +} + +#[derive(Clone, PartialEq)] +pub struct OcrOutcome { + pub normalized: NormalizedOcr, + pub native: NativeOcrPayload, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn normalized_serialization_excludes_the_native_payload() { + let outcome = OcrOutcome { + normalized: NormalizedOcr { + pages: vec![OcrPage { + index: 0, + markdown: "portable".to_string(), + }], + model: "ocr-model".to_string(), + document_annotation: None, + content: None, + tables: None, + key_value_pairs: None, + usage: OcrUsage::default(), + }, + native: NativeOcrPayload::new(json!({"provider_secret_field": "native"})), + }; + let public = + serde_json::to_value(&outcome.normalized).expect("normalized output serializes"); + assert_eq!(public["pages"][0]["markdown"], "portable"); + assert!(public.get("provider_secret_field").is_none()); + assert_eq!(outcome.native.as_value()["provider_secret_field"], "native"); + } +} diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs index 1a72b8f1d66..e5ed78987e8 100644 --- a/litellm-rust/crates/core/src/ocr/types.rs +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -1,6 +1,34 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Field { + Absent, + Null, + Value(T), +} + +impl Field { + pub fn as_ref(&self) -> Field<&T> { + match self { + Self::Absent => Field::Absent, + Self::Null => Field::Null, + Self::Value(value) => Field::Value(value), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OcrDialectId { + Mistral, + AzureFoundryMistral, + AzureDocumentIntelligence, + VertexMistral, + VertexDeepSeek, + ReductoV3, + ReductoLegacy, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct OcrRequestData { pub data: Value, From 3e3d3ce32985208fb32dc2a538c60a99742c7504 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 06:54:26 -0700 Subject: [PATCH 66/67] feat(rust): add OCR compiler plans and dialect profiles --- litellm-rust/Cargo.lock | 1 + litellm-rust/crates/core/Cargo.toml | 1 + litellm-rust/crates/core/src/ocr/compiler.rs | 218 +++++++++++ litellm-rust/crates/core/src/ocr/mod.rs | 4 + litellm-rust/crates/core/src/ocr/plan.rs | 40 +++ litellm-rust/crates/core/src/ocr/profile.rs | 189 ++++++++++ litellm-rust/crates/core/src/ocr/wire.rs | 358 +++++++++++++++++++ 7 files changed, 811 insertions(+) create mode 100644 litellm-rust/crates/core/src/ocr/compiler.rs create mode 100644 litellm-rust/crates/core/src/ocr/plan.rs create mode 100644 litellm-rust/crates/core/src/ocr/profile.rs create mode 100644 litellm-rust/crates/core/src/ocr/wire.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 603047db7dc..ac9f70e33d7 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1428,6 +1428,7 @@ dependencies = [ "aws-sigv4", "aws-smithy-runtime-api", "aws-types", + "base64", "bytes", "futures-util", "mime", diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 46e22bc866a..3eb9da53277 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true futures-util.workspace = true bytes.workspace = true mime.workspace = true diff --git a/litellm-rust/crates/core/src/ocr/compiler.rs b/litellm-rust/crates/core/src/ocr/compiler.rs new file mode 100644 index 00000000000..331132d9ceb --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/compiler.rs @@ -0,0 +1,218 @@ +use std::collections::BTreeMap; + +use serde_json::Value; +use thiserror::Error; +use url::Url; + +use super::canonical::{CanonicalOcrRequest, DocumentKind, OcrDocument}; +use super::plan::{CompletionPlan, DocumentPlan}; +use super::policy::OcrParameterPolicy; +use super::response::NormalizedOcr; +use super::types::OcrDialectId; +pub use super::wire::{MultipartBodyPlan, MultipartPart, OcrJsonValue, OcrWireBody, OcrWireError}; + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum CompileError { + #[error("invalid OCR parameter {field}: {reason}")] + InvalidParameter { + field: &'static str, + reason: &'static str, + }, + #[error("OCR dialect is not compiled yet: {0:?}")] + UnsupportedDialect(OcrDialectId), +} + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum NormalizeError { + #[error("invalid terminal OCR response: {0}")] + InvalidPayload(&'static str), +} + +#[derive(Clone, Default, PartialEq, Eq)] +pub struct OcrCredentials { + api_key: Option, + oauth_token: Option, +} + +impl OcrCredentials { + pub fn new(api_key: Option, oauth_token: Option) -> Self { + Self { + api_key, + oauth_token, + } + } + + pub fn api_key(&self) -> Option<&str> { + self.api_key.as_deref() + } + + pub fn oauth_token(&self) -> Option<&str> { + self.oauth_token.as_deref() + } +} + +#[derive(Clone, PartialEq)] +pub struct ResolvedOcrTarget { + dialect: OcrDialectId, + api_base: Url, + credentials: OcrCredentials, +} + +impl ResolvedOcrTarget { + pub fn new(dialect: OcrDialectId, api_base: Url, credentials: OcrCredentials) -> Self { + Self { + dialect, + api_base, + credentials, + } + } + + pub fn dialect(&self) -> OcrDialectId { + self.dialect + } + + pub fn api_base(&self) -> &Url { + &self.api_base + } + + pub fn credentials(&self) -> &OcrCredentials { + &self.credentials + } +} + +#[derive(Clone, PartialEq)] +pub enum ProviderDocument { + RemoteUrl { + kind: DocumentKind, + url: Url, + }, + Inline { + kind: DocumentKind, + media_type: mime::Mime, + bytes: bytes::Bytes, + }, + Reference { + id: String, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HttpMethod { + Get, + Post, +} + +#[derive(Clone, PartialEq)] +pub struct CompiledHttpRequest { + pub method: HttpMethod, + pub url: Url, + pub headers: BTreeMap, + pub body: OcrWireBody, +} + +#[derive(Clone, PartialEq)] +pub struct ProviderPayload(Value); + +impl ProviderPayload { + pub fn new(value: Value) -> Self { + Self(value) + } + + pub fn as_value(&self) -> &Value { + &self.0 + } + + pub fn into_value(self) -> Value { + self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OcrDocumentPolicy { + Ready, + FetchRemoteUrlAndInline, + UploadUnlessProviderReference, +} + +pub trait OcrDialectCompiler: Send + Sync { + fn parameter_policy(&self) -> &'static OcrParameterPolicy; + + fn prepare_document( + &self, + document: &OcrDocument, + target: &ResolvedOcrTarget, + ) -> Result; + + fn compile_submit( + &self, + request: &CanonicalOcrRequest, + document: ProviderDocument, + target: &ResolvedOcrTarget, + ) -> Result; + + fn completion_plan(&self) -> CompletionPlan; + + fn normalize( + &self, + terminal_response: ProviderPayload, + ) -> Result; +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use mime::Mime; + use serde_json::json; + + use super::*; + + #[test] + fn compilation_preserves_the_inline_media_allocation() { + let source = Bytes::from_static(b"pdf payload"); + let source_pointer = source.as_ptr(); + let canonical_document = OcrDocument::Inline { + kind: DocumentKind::Pdf, + media_type: "application/pdf".parse::().expect("valid MIME type"), + bytes: source.clone(), + }; + let OcrDocument::Inline { + kind, + media_type, + bytes, + } = &canonical_document + else { + panic!("inline canonical document expected"); + }; + let provider_document = ProviderDocument::Inline { + kind: *kind, + media_type: media_type.clone(), + bytes: bytes.clone(), + }; + let ProviderDocument::Inline { + media_type, bytes, .. + } = provider_document + else { + panic!("inline document expected"); + }; + let request = CompiledHttpRequest { + method: HttpMethod::Post, + url: Url::parse("https://example.com/ocr").expect("valid URL"), + headers: BTreeMap::from([("content-type".to_string(), "application/json".to_string())]), + body: OcrWireBody::JsonWithMedia(OcrJsonValue::Object(BTreeMap::from([ + ( + "document".to_string(), + OcrJsonValue::InlineDataUri { media_type, bytes }, + ), + ("model".to_string(), OcrJsonValue::Value(json!("ocr-model"))), + ]))), + }; + + let OcrWireBody::JsonWithMedia(OcrJsonValue::Object(fields)) = &request.body else { + panic!("media JSON body expected"); + }; + let OcrJsonValue::InlineDataUri { bytes, .. } = &fields["document"] else { + panic!("inline media expected"); + }; + assert_eq!(bytes.as_ptr(), source_pointer); + } +} diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index d28291a971e..14ab6f2294e 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -1,5 +1,9 @@ pub mod canonical; +pub mod compiler; +pub mod plan; pub mod policy; +pub mod profile; pub mod response; pub mod transformation; pub mod types; +pub mod wire; diff --git a/litellm-rust/crates/core/src/ocr/plan.rs b/litellm-rust/crates/core/src/ocr/plan.rs new file mode 100644 index 00000000000..724f83e5a63 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/plan.rs @@ -0,0 +1,40 @@ +use std::time::Duration; + +use url::Url; + +use super::canonical::{DocumentKind, OcrDocument}; +use super::compiler::ProviderDocument; +use super::types::OcrDialectId; + +#[derive(Clone, PartialEq)] +pub enum DocumentPlan { + Ready(ProviderDocument), + FetchAndInline(FetchPlan), + Upload(UploadPlan), +} + +#[derive(Clone, PartialEq)] +pub struct FetchPlan { + pub kind: DocumentKind, + pub url: Url, + pub max_bytes: usize, +} + +#[derive(Clone, PartialEq)] +pub struct UploadPlan { + pub dialect: OcrDialectId, + pub document: OcrDocument, +} + +#[derive(Clone, PartialEq)] +pub enum CompletionPlan { + Immediate, + Poll(PollPlan), +} + +#[derive(Clone, PartialEq, Eq)] +pub struct PollPlan { + pub operation_location_header: &'static str, + pub interval: Duration, + pub timeout: Duration, +} diff --git a/litellm-rust/crates/core/src/ocr/profile.rs b/litellm-rust/crates/core/src/ocr/profile.rs new file mode 100644 index 00000000000..af1637eaada --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/profile.rs @@ -0,0 +1,189 @@ +use super::compiler::OcrDocumentPolicy; +use super::policy::{OcrParameterPolicy, ParameterDisposition}; +use super::types::OcrDialectId; + +pub const MISTRAL_OCR_PARAMETER_POLICY: OcrParameterPolicy = OcrParameterPolicy { + pages: ParameterDisposition::Forward, + include_image_base64: ParameterDisposition::Forward, + image_limit: ParameterDisposition::Forward, + image_min_size: ParameterDisposition::Forward, + bbox_annotation_format: ParameterDisposition::Forward, + document_annotation_format: ParameterDisposition::Forward, + document_annotation_prompt: ParameterDisposition::Forward, + extract_header: ParameterDisposition::Forward, + extract_footer: ParameterDisposition::Forward, + table_format: ParameterDisposition::Forward, + confidence_scores_granularity: ParameterDisposition::Forward, + include_blocks: ParameterDisposition::Forward, + request_id: ParameterDisposition::Forward, +}; + +pub const AZURE_DOCUMENT_INTELLIGENCE_PARAMETER_POLICY: OcrParameterPolicy = OcrParameterPolicy { + pages: ParameterDisposition::Transform, + include_image_base64: ParameterDisposition::Reject, + image_limit: ParameterDisposition::Reject, + image_min_size: ParameterDisposition::Reject, + bbox_annotation_format: ParameterDisposition::Reject, + document_annotation_format: ParameterDisposition::Reject, + document_annotation_prompt: ParameterDisposition::Reject, + extract_header: ParameterDisposition::Reject, + extract_footer: ParameterDisposition::Reject, + table_format: ParameterDisposition::Reject, + confidence_scores_granularity: ParameterDisposition::Reject, + include_blocks: ParameterDisposition::Reject, + request_id: ParameterDisposition::Reject, +}; + +pub const REJECT_CANONICAL_OCR_PARAMETER_POLICY: OcrParameterPolicy = OcrParameterPolicy { + pages: ParameterDisposition::Reject, + include_image_base64: ParameterDisposition::Reject, + image_limit: ParameterDisposition::Reject, + image_min_size: ParameterDisposition::Reject, + bbox_annotation_format: ParameterDisposition::Reject, + document_annotation_format: ParameterDisposition::Reject, + document_annotation_prompt: ParameterDisposition::Reject, + extract_header: ParameterDisposition::Reject, + extract_footer: ParameterDisposition::Reject, + table_format: ParameterDisposition::Reject, + confidence_scores_granularity: ParameterDisposition::Reject, + include_blocks: ParameterDisposition::Reject, + request_id: ParameterDisposition::Reject, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct OcrPollingProfile { + pub operation_location_header: &'static str, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct OcrDialectProfile { + pub dialect: OcrDialectId, + pub parameter_policy: &'static OcrParameterPolicy, + pub document_policy: OcrDocumentPolicy, + pub polling: Option, +} + +pub const OCR_DIALECT_PROFILES: [OcrDialectProfile; 7] = [ + OcrDialectProfile { + dialect: OcrDialectId::Mistral, + parameter_policy: &MISTRAL_OCR_PARAMETER_POLICY, + document_policy: OcrDocumentPolicy::Ready, + polling: None, + }, + OcrDialectProfile { + dialect: OcrDialectId::AzureFoundryMistral, + parameter_policy: &MISTRAL_OCR_PARAMETER_POLICY, + document_policy: OcrDocumentPolicy::FetchRemoteUrlAndInline, + polling: None, + }, + OcrDialectProfile { + dialect: OcrDialectId::AzureDocumentIntelligence, + parameter_policy: &AZURE_DOCUMENT_INTELLIGENCE_PARAMETER_POLICY, + document_policy: OcrDocumentPolicy::Ready, + polling: Some(OcrPollingProfile { + operation_location_header: "operation-location", + }), + }, + OcrDialectProfile { + dialect: OcrDialectId::VertexMistral, + parameter_policy: &MISTRAL_OCR_PARAMETER_POLICY, + document_policy: OcrDocumentPolicy::FetchRemoteUrlAndInline, + polling: None, + }, + OcrDialectProfile { + dialect: OcrDialectId::VertexDeepSeek, + parameter_policy: &REJECT_CANONICAL_OCR_PARAMETER_POLICY, + document_policy: OcrDocumentPolicy::Ready, + polling: None, + }, + OcrDialectProfile { + dialect: OcrDialectId::ReductoV3, + parameter_policy: &REJECT_CANONICAL_OCR_PARAMETER_POLICY, + document_policy: OcrDocumentPolicy::UploadUnlessProviderReference, + polling: None, + }, + OcrDialectProfile { + dialect: OcrDialectId::ReductoLegacy, + parameter_policy: &REJECT_CANONICAL_OCR_PARAMETER_POLICY, + document_policy: OcrDocumentPolicy::UploadUnlessProviderReference, + polling: None, + }, +]; + +pub const fn ocr_dialect_profile(dialect: OcrDialectId) -> &'static OcrDialectProfile { + match dialect { + OcrDialectId::Mistral => &OCR_DIALECT_PROFILES[0], + OcrDialectId::AzureFoundryMistral => &OCR_DIALECT_PROFILES[1], + OcrDialectId::AzureDocumentIntelligence => &OCR_DIALECT_PROFILES[2], + OcrDialectId::VertexMistral => &OCR_DIALECT_PROFILES[3], + OcrDialectId::VertexDeepSeek => &OCR_DIALECT_PROFILES[4], + OcrDialectId::ReductoV3 => &OCR_DIALECT_PROFILES[5], + OcrDialectId::ReductoLegacy => &OCR_DIALECT_PROFILES[6], + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ocr::policy::{OcrCanonicalField, ParameterDisposition}; + + #[test] + fn mistral_compatible_dialects_share_parameter_rules_but_not_document_rules() { + let mistral = ocr_dialect_profile(OcrDialectId::Mistral); + let foundry = ocr_dialect_profile(OcrDialectId::AzureFoundryMistral); + let vertex = ocr_dialect_profile(OcrDialectId::VertexMistral); + + for field in OcrCanonicalField::ALL { + let expected = mistral.parameter_policy.disposition(field); + assert_eq!(foundry.parameter_policy.disposition(field), expected); + assert_eq!(vertex.parameter_policy.disposition(field), expected); + } + assert_eq!(mistral.document_policy, OcrDocumentPolicy::Ready); + assert_eq!( + foundry.document_policy, + OcrDocumentPolicy::FetchRemoteUrlAndInline + ); + assert_eq!( + vertex.document_policy, + OcrDocumentPolicy::FetchRemoteUrlAndInline + ); + } + + #[test] + fn non_mistral_profiles_preserve_provider_specific_boundaries() { + let azure = ocr_dialect_profile(OcrDialectId::AzureDocumentIntelligence); + assert_eq!( + azure.parameter_policy.disposition(OcrCanonicalField::Pages), + ParameterDisposition::Transform + ); + assert_eq!( + azure.polling, + Some(OcrPollingProfile { + operation_location_header: "operation-location" + }) + ); + + for dialect in [OcrDialectId::ReductoV3, OcrDialectId::ReductoLegacy] { + let reducto = ocr_dialect_profile(dialect); + assert_eq!( + reducto.document_policy, + OcrDocumentPolicy::UploadUnlessProviderReference + ); + assert!(OcrCanonicalField::ALL.iter().all(|field| { + reducto.parameter_policy.disposition(*field) == ParameterDisposition::Reject + })); + } + } + + #[test] + fn every_dialect_has_exactly_one_profile() { + for (index, profile) in OCR_DIALECT_PROFILES.iter().enumerate() { + assert_eq!(ocr_dialect_profile(profile.dialect), profile); + assert!( + OCR_DIALECT_PROFILES[index + 1..] + .iter() + .all(|other| other.dialect != profile.dialect) + ); + } + } +} diff --git a/litellm-rust/crates/core/src/ocr/wire.rs b/litellm-rust/crates/core/src/ocr/wire.rs new file mode 100644 index 00000000000..e920e96519b --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/wire.rs @@ -0,0 +1,358 @@ +use std::collections::BTreeMap; +use std::io::{self, Write}; +use std::str::Utf8Error; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use bytes::Bytes; +use mime::Mime; +use serde_json::Value; +use thiserror::Error; + +const BASE64_INPUT_CHUNK_SIZE: usize = 48 * 1024; + +#[derive(Debug, Error)] +pub enum OcrWireError { + #[error("failed to encode OCR JSON body: {0}")] + Json(#[from] serde_json::Error), + #[error("failed to write OCR body: {0}")] + Io(#[from] io::Error), + #[error("encoded OCR data URI is not UTF-8: {0}")] + InvalidDataUri(#[from] Utf8Error), +} + +#[derive(Clone, PartialEq)] +pub enum OcrJsonValue { + Value(Value), + Array(Vec), + Object(BTreeMap), + InlineDataUri { media_type: Mime, bytes: Bytes }, + EncodedDataUri(Bytes), +} + +impl OcrJsonValue { + fn write_to(&self, writer: &mut impl Write) -> Result<(), OcrWireError> { + match self { + Self::Value(value) => serde_json::to_writer(writer, value).map_err(Into::into), + Self::Array(values) => { + writer.write_all(b"[")?; + for (index, value) in values.iter().enumerate() { + if index != 0 { + writer.write_all(b",")?; + } + value.write_to(writer)?; + } + writer.write_all(b"]")?; + Ok(()) + } + Self::Object(fields) => { + writer.write_all(b"{")?; + for (index, (key, value)) in fields.iter().enumerate() { + if index != 0 { + writer.write_all(b",")?; + } + serde_json::to_writer(&mut *writer, key)?; + writer.write_all(b":")?; + value.write_to(writer)?; + } + writer.write_all(b"}")?; + Ok(()) + } + Self::InlineDataUri { media_type, bytes } => { + writer.write_all(b"\"data:")?; + writer.write_all(media_type.as_ref().as_bytes())?; + writer.write_all(b";base64,")?; + for chunk in bytes.chunks(BASE64_INPUT_CHUNK_SIZE) { + let encoded = STANDARD.encode(chunk); + writer.write_all(encoded.as_bytes())?; + } + writer.write_all(b"\"")?; + Ok(()) + } + Self::EncodedDataUri(data_uri) => { + let data_uri = std::str::from_utf8(data_uri)?; + serde_json::to_writer(writer, data_uri).map_err(Into::into) + } + } + } +} + +#[derive(Clone, PartialEq)] +pub enum MultipartPart { + Text { + name: String, + value: String, + }, + Json { + name: String, + value: Value, + }, + File { + name: String, + file_name: String, + media_type: Mime, + bytes: Bytes, + }, +} + +#[derive(Clone, PartialEq)] +pub struct MultipartBodyPlan { + boundary: String, + parts: Vec, +} + +impl MultipartBodyPlan { + pub fn new(boundary: impl Into, parts: Vec) -> Self { + Self { + boundary: boundary.into(), + parts, + } + } + + pub fn boundary(&self) -> &str { + &self.boundary + } + + pub fn parts(&self) -> &[MultipartPart] { + &self.parts + } + + fn write_to(&self, writer: &mut impl Write) -> Result<(), OcrWireError> { + for part in &self.parts { + write!(writer, "--{}\r\n", self.boundary)?; + match part { + MultipartPart::Text { name, value } => { + write!( + writer, + "Content-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n" + )?; + } + MultipartPart::Json { name, value } => { + write!( + writer, + "Content-Disposition: form-data; name=\"{name}\"\r\nContent-Type: application/json\r\n\r\n" + )?; + serde_json::to_writer(&mut *writer, value)?; + writer.write_all(b"\r\n")?; + } + MultipartPart::File { + name, + file_name, + media_type, + bytes, + } => { + write!( + writer, + "Content-Disposition: form-data; name=\"{name}\"; filename=\"{file_name}\"\r\nContent-Type: {media_type}\r\n\r\n" + )?; + writer.write_all(bytes)?; + writer.write_all(b"\r\n")?; + } + } + } + write!(writer, "--{}--\r\n", self.boundary)?; + Ok(()) + } +} + +#[derive(Clone, PartialEq)] +pub enum OcrWireBody { + Json(Value), + JsonWithMedia(OcrJsonValue), + Multipart(MultipartBodyPlan), +} + +impl OcrWireBody { + pub fn content_type(&self) -> String { + match self { + Self::Json(_) | Self::JsonWithMedia(_) => "application/json".to_string(), + Self::Multipart(plan) => { + format!("multipart/form-data; boundary={}", plan.boundary()) + } + } + } + + pub fn write_to(&self, mut writer: impl Write) -> Result<(), OcrWireError> { + match self { + Self::Json(value) => serde_json::to_writer(writer, value).map_err(Into::into), + Self::JsonWithMedia(value) => value.write_to(&mut writer), + Self::Multipart(plan) => plan.write_to(&mut writer), + } + } +} + +#[cfg(test)] +mod tests { + use std::io; + use std::sync::Arc; + + use serde_json::json; + + use super::*; + + #[test] + fn ordinary_json_body_writes_without_a_media_plan() { + let body = OcrWireBody::Json(json!({"model": "ocr-model", "pages": [0, 2]})); + let mut encoded = Vec::new(); + + body.write_to(&mut encoded).expect("body writes"); + + assert_eq!(body.content_type(), "application/json"); + assert_eq!( + serde_json::from_slice::(&encoded).expect("valid JSON"), + json!({"model": "ocr-model", "pages": [0, 2]}) + ); + } + + #[test] + fn json_with_media_streams_raw_bytes_as_a_data_uri() { + let owner: Arc<[u8]> = vec![b'x'; BASE64_INPUT_CHUNK_SIZE + 1].into(); + let bytes = Bytes::from_owner(Arc::clone(&owner)); + let source_pointer = bytes.as_ptr(); + let body = OcrWireBody::JsonWithMedia(OcrJsonValue::Object(BTreeMap::from([ + ( + "document".to_string(), + OcrJsonValue::InlineDataUri { + media_type: "application/pdf".parse().expect("valid MIME type"), + bytes, + }, + ), + ("model".to_string(), OcrJsonValue::Value(json!("ocr-model"))), + ]))); + + let OcrWireBody::JsonWithMedia(OcrJsonValue::Object(fields)) = &body else { + panic!("media JSON body must retain its typed representation"); + }; + let OcrJsonValue::InlineDataUri { bytes, .. } = &fields["document"] else { + panic!("document must remain shared binary media"); + }; + assert_eq!(bytes.as_ptr(), source_pointer); + assert_eq!(Arc::strong_count(&owner), 2); + + let mut encoded = Vec::new(); + body.write_to(&mut encoded).expect("body writes"); + let expected_data_uri = format!( + "data:application/pdf;base64,{}", + STANDARD.encode(owner.as_ref()) + ); + let expected = json!({"document": expected_data_uri, "model": "ocr-model"}); + assert_eq!( + serde_json::from_slice::(&encoded).expect("valid JSON"), + expected + ); + } + + #[test] + fn raw_media_encoding_uses_bounded_writes() { + let bytes = Bytes::from(vec![b'x'; BASE64_INPUT_CHUNK_SIZE * 3 + 1]); + let body = OcrWireBody::JsonWithMedia(OcrJsonValue::InlineDataUri { + media_type: "application/pdf".parse().expect("valid MIME type"), + bytes, + }); + let mut sink = BoundedSink { + maximum_write: BASE64_INPUT_CHUNK_SIZE * 4 / 3, + written: 0, + }; + + body.write_to(&mut sink).expect("writes remain bounded"); + + assert!(sink.written > BASE64_INPUT_CHUNK_SIZE * 4); + } + + #[test] + fn encoded_data_uri_is_retained_without_decoding_or_copying() { + let data_uri = Bytes::from_static(b"data:image/png;base64,aGVsbG8="); + let source_pointer = data_uri.as_ptr(); + let body = OcrWireBody::JsonWithMedia(OcrJsonValue::EncodedDataUri(data_uri)); + + let OcrWireBody::JsonWithMedia(OcrJsonValue::EncodedDataUri(retained)) = &body else { + panic!("encoded data URI must remain bytes"); + }; + assert_eq!(retained.as_ptr(), source_pointer); + + let mut encoded = Vec::new(); + body.write_to(&mut encoded).expect("body writes"); + assert_eq!(encoded, br#""data:image/png;base64,aGVsbG8=""#); + } + + #[test] + fn encoded_data_uri_is_json_escaped_without_changing_its_allocation() { + let data_uri = Bytes::from_static(b"data:text/plain,quoted%20\"value\""); + let source_pointer = data_uri.as_ptr(); + let body = OcrWireBody::JsonWithMedia(OcrJsonValue::EncodedDataUri(data_uri)); + + let mut encoded = Vec::new(); + body.write_to(&mut encoded).expect("body writes"); + + let OcrWireBody::JsonWithMedia(OcrJsonValue::EncodedDataUri(retained)) = &body else { + panic!("encoded data URI expected"); + }; + assert_eq!(retained.as_ptr(), source_pointer); + assert_eq!( + serde_json::from_slice::(&encoded).expect("valid JSON string"), + "data:text/plain,quoted%20\"value\"" + ); + } + + #[test] + fn multipart_file_is_replayable_and_retains_shared_bytes() { + let file = Bytes::from_static(b"large-pdf-payload"); + let source_pointer = file.as_ptr(); + let body = OcrWireBody::Multipart(MultipartBodyPlan::new( + "ocr-boundary", + vec![MultipartPart::File { + name: "file".to_string(), + file_name: "document.pdf".to_string(), + media_type: "application/pdf".parse().expect("valid MIME type"), + bytes: file, + }], + )); + + let OcrWireBody::Multipart(plan) = &body else { + panic!("multipart body expected"); + }; + let MultipartPart::File { bytes, .. } = &plan.parts()[0] else { + panic!("file part expected"); + }; + assert_eq!(bytes.as_ptr(), source_pointer); + + let mut first = Vec::new(); + let mut retry = Vec::new(); + body.write_to(&mut first).expect("first write succeeds"); + body.write_to(&mut retry).expect("retry write succeeds"); + assert_eq!(first, retry); + assert!( + first + .windows(file_name_marker().len()) + .any(|window| window == file_name_marker()) + ); + assert!( + first + .windows(bytes.len()) + .any(|window| window == bytes.as_ref()) + ); + } + + fn file_name_marker() -> &'static [u8] { + b"filename=\"document.pdf\"" + } + + struct BoundedSink { + maximum_write: usize, + written: usize, + } + + impl Write for BoundedSink { + fn write(&mut self, buffer: &[u8]) -> io::Result { + if buffer.len() > self.maximum_write { + return Err(io::Error::other("write exceeded bound")); + } + self.written += buffer.len(); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } +} From 112611c9aa90a14f48b89762151463cdeb292ba4 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 2 Sep 2026 11:36:04 -0700 Subject: [PATCH 67/67] test(ocr): add recorded fixture parity harness atop Rust bridge stack --- .github/workflows/test-unit.yml | 1 + litellm-rust/Cargo.toml | 2 +- .../providers/azure_ai/ocr/transformation.rs | 146 +- .../crates/python-bridge/src/errors.rs | 33 + .../crates/python-bridge/src/routes/ocr.rs | 4 +- litellm/ocr/main.py | 10 +- litellm/rust_bridge/ocr.py | 24 +- litellm/rust_bridge/runtime.py | 15 +- pyproject.toml | 2 + tests/code_coverage_tests/liccheck.ini | 1 + tests/e2e/provider_edge.py | 38 +- tests/provider_record_replay/__init__.py | 1 + tests/provider_record_replay/http.py | 54 + tests/route_parity/README.md | 91 ++ tests/route_parity/__init__.py | 3 + tests/route_parity/compare.py | 80 + tests/route_parity/fixture_models.py | 64 + tests/route_parity/fixtures/__init__.py | 1 + tests/route_parity/fixtures/cassette.py | 147 ++ tests/route_parity/fixtures/cli.py | 34 + tests/route_parity/fixtures/inputs.py | 22 + tests/route_parity/fixtures/media.py | 225 +++ tests/route_parity/fixtures/pipeline.py | 198 +++ tests/route_parity/fixtures/pytest_support.py | 66 + tests/route_parity/fixtures/recording.py | 267 ++++ tests/route_parity/fixtures/store.py | 126 ++ tests/route_parity/fixtures/test_cassette.py | 95 ++ tests/route_parity/fixtures/test_inputs.py | 20 + tests/route_parity/fixtures/test_media.py | 27 + tests/route_parity/fixtures/test_pipeline.py | 191 +++ tests/route_parity/fixtures/test_recording.py | 574 +++++++ tests/route_parity/inprocess.py | 47 + tests/route_parity/models.py | 145 ++ tests/route_parity/recorded_http.py | 63 + tests/route_parity/replay.py | 147 ++ tests/route_parity/runner.py | 195 +++ tests/route_parity/stream.py | 177 +++ tests/route_parity/test_parity.py | 190 +++ tests/route_parity/test_stream.py | 344 ++++ tests/test_litellm/ocr/conftest.py | 43 + tests/test_litellm/ocr/fixtures/README.md | 49 + tests/test_litellm/ocr/fixtures/__init__.py | 1 + tests/test_litellm/ocr/fixtures/azure.py | 207 +++ tests/test_litellm/ocr/fixtures/base.py | 48 + tests/test_litellm/ocr/fixtures/common.py | 101 ++ tests/test_litellm/ocr/fixtures/config.py | 13 + ...2f2996578fff02d93ec3011031ce2b4c511fc.yaml | 68 + ...c7ded7fea981215a478a12d8636eff8eb1157.yaml | 60 + ...b119bced6537f45cf08b27cbec08bb8f45951.yaml | 69 + ...f46da887570de720a142e9df297bdd280608d.yaml | 85 + ...625aa688e6cbd69e01f94d1f3205ef41ab830.yaml | 68 + ...01606b7039583f55a0f85bfe38c7c15712adf.yaml | 67 + ...da532390140ade4e10b5bb07570ca7b372499.yaml | 82 + ...598ef9e6df1813bee21137e88bcc14ce7a473.yaml | 92 ++ ...358dc7c9b0eafc64e7e2516d46e290fc93092.yaml | 83 + ...57cd562048fc6a993cba093fe86a8937d104b.yaml | 68 + ...e88e27651580bac1e145ad352728b1a0acd31.yaml | 68 + ...8047b28006b5cbd4486d54c6fe89070e7a671.yaml | 68 + ...d77447359bba8c29d6b31e650255298271d47.yaml | 67 + ...3bba41a29b707d0cfafd74986e6ad8c2ed900.yaml | 68 + ...b3b7dc37c5bf5dfd47e873c4cc695d6d7461f.yaml | 68 + ...689c3583b09694a7ac086e4707be49b35eb06.yaml | 83 + ...ae74dd363ddda0d6677155afbd145d0dc3c36.yaml | 68 + ...6ab165f1866eb074a8c079273eef5df512985.yaml | 105 ++ ...030105ad107ae3400532b20ad70ebbd021954.yaml | 70 + ...7e201dfb9ac31de66094734ef38c288293625.yaml | 39 + ...37e51b7e915b96a85e4ce8dfc41ba5f7f11df.yaml | 70 + ...7e3b8fdbfe1fb5a9818a2fe47f02c2aad0fac.yaml | 69 + ...7398c1c26cf1fb8f134b1326e79bd3d487c41.yaml | 72 + ...0c95f731a0913d223fbe1780bbaaef1ecd835.yaml | 83 + ...e4239015b9b44376ce49c921f8fc057fb9c6a.yaml | 70 + ...653fe30bde512da747d2fd8e78759ccdc9b90.yaml | 70 + ...b98a3a0d251a27c3bb80178580a711c111e94.yaml | 100 ++ ...3999653d141626ca1eadfdb47fc20d645b9d5.yaml | 72 + ...84da6aa7892f8cdd4f22158ba4c700dbe8a81.yaml | 39 + ...e92b04e51b1ea8b74eaa4a4382a01154b5f5a.yaml | 71 + ...97f703d67d234593ac188f948142190fdf06d.yaml | 69 + tests/test_litellm/ocr/fixtures/migrate.py | 63 + tests/test_litellm/ocr/fixtures/mistral.py | 259 +++ tests/test_litellm/ocr/fixtures/models.py | 71 + tests/test_litellm/ocr/fixtures/record.py | 75 + tests/test_litellm/ocr/fixtures/reducto.py | 440 ++++++ tests/test_litellm/ocr/fixtures/vertex.py | 148 ++ tests/test_litellm/ocr/test_fixture_models.py | 1388 +++++++++++++++++ tests/test_litellm/ocr/test_fixture_store.py | 85 + .../test_litellm/ocr/test_record_fixtures.py | 366 +++++ tests/test_litellm/ocr/test_rust_bridge.py | 59 +- tests/test_litellm/ocr/test_sdk_parity.py | 463 ++++++ .../common_utils/test_user_api_key_cache.py | 2 +- .../test_litellm/rust_bridge/test_runtime.py | 57 + uv.lock | 109 ++ 91 files changed, 10102 insertions(+), 46 deletions(-) create mode 100644 tests/provider_record_replay/__init__.py create mode 100644 tests/provider_record_replay/http.py create mode 100644 tests/route_parity/README.md create mode 100644 tests/route_parity/__init__.py create mode 100644 tests/route_parity/compare.py create mode 100644 tests/route_parity/fixture_models.py create mode 100644 tests/route_parity/fixtures/__init__.py create mode 100644 tests/route_parity/fixtures/cassette.py create mode 100644 tests/route_parity/fixtures/cli.py create mode 100644 tests/route_parity/fixtures/inputs.py create mode 100644 tests/route_parity/fixtures/media.py create mode 100644 tests/route_parity/fixtures/pipeline.py create mode 100644 tests/route_parity/fixtures/pytest_support.py create mode 100644 tests/route_parity/fixtures/recording.py create mode 100644 tests/route_parity/fixtures/store.py create mode 100644 tests/route_parity/fixtures/test_cassette.py create mode 100644 tests/route_parity/fixtures/test_inputs.py create mode 100644 tests/route_parity/fixtures/test_media.py create mode 100644 tests/route_parity/fixtures/test_pipeline.py create mode 100644 tests/route_parity/fixtures/test_recording.py create mode 100644 tests/route_parity/inprocess.py create mode 100644 tests/route_parity/models.py create mode 100644 tests/route_parity/recorded_http.py create mode 100644 tests/route_parity/replay.py create mode 100644 tests/route_parity/runner.py create mode 100644 tests/route_parity/stream.py create mode 100644 tests/route_parity/test_parity.py create mode 100644 tests/route_parity/test_stream.py create mode 100644 tests/test_litellm/ocr/conftest.py create mode 100644 tests/test_litellm/ocr/fixtures/README.md create mode 100644 tests/test_litellm/ocr/fixtures/__init__.py create mode 100644 tests/test_litellm/ocr/fixtures/azure.py create mode 100644 tests/test_litellm/ocr/fixtures/base.py create mode 100644 tests/test_litellm/ocr/fixtures/common.py create mode 100644 tests/test_litellm/ocr/fixtures/config.py create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/0b234402bdfd3be223e731051202f2996578fff02d93ec3011031ce2b4c511fc.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/1f3001924ef2c46d3eccdedd808c7ded7fea981215a478a12d8636eff8eb1157.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/1f38e1eab857c22107df833aa63b119bced6537f45cf08b27cbec08bb8f45951.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/38dbc05d8508355879de894ecc0f46da887570de720a142e9df297bdd280608d.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/3d5f42192461789f547b42c8f1b625aa688e6cbd69e01f94d1f3205ef41ab830.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/54ce0991a21ea668e2373d2b05801606b7039583f55a0f85bfe38c7c15712adf.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/7f7dc004625cb90e3f4bd2f473fda532390140ade4e10b5bb07570ca7b372499.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/99d9d5ac0449213a562a36163ee598ef9e6df1813bee21137e88bcc14ce7a473.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/9b9fb2f3a7eb8ca1e2d128ea96b358dc7c9b0eafc64e7e2516d46e290fc93092.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/a8693352d9ce0d7d120b2351d2257cd562048fc6a993cba093fe86a8937d104b.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/b19fdc140375a4eadfd6eb4e4cbe88e27651580bac1e145ad352728b1a0acd31.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/bdb616d21dece89481debccb9358047b28006b5cbd4486d54c6fe89070e7a671.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/c3066e2f2f964725b71ae891821d77447359bba8c29d6b31e650255298271d47.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/c57601c90e9b289e506cdc818523bba41a29b707d0cfafd74986e6ad8c2ed900.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/c5d33cb2b20fb76c9543ff57dc9b3b7dc37c5bf5dfd47e873c4cc695d6d7461f.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/c6f0f158205be7fc00bd962e341689c3583b09694a7ac086e4707be49b35eb06.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/cea250810b49cd17251ee8c036eae74dd363ddda0d6677155afbd145d0dc3c36.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/mistral-ocr/d80a6ccca02922ba9094426235a6ab165f1866eb074a8c079273eef5df512985.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-legacy/09cc2f0155ffdc3d25abff96797030105ad107ae3400532b20ad70ebbd021954.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-legacy/7c4bbc65dfbc17bac7f0d179c5d7e201dfb9ac31de66094734ef38c288293625.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-legacy/953d2a3bfea5594f9df19fc3f6c37e51b7e915b96a85e4ce8dfc41ba5f7f11df.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-legacy/a41d4c9547448df927b308d619d7e3b8fdbfe1fb5a9818a2fe47f02c2aad0fac.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-v3/2148dfafc823aea933d7c9aa8877398c1c26cf1fb8f134b1326e79bd3d487c41.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-v3/50890748074024915ef138fcb8e0c95f731a0913d223fbe1780bbaaef1ecd835.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-v3/7cefa9e08179db20d45e255d269e4239015b9b44376ce49c921f8fc057fb9c6a.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-v3/8270ee13cdfb2ceefd8302a1880653fe30bde512da747d2fd8e78759ccdc9b90.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-v3/997cd381eb3019b71086e4fdab1b98a3a0d251a27c3bb80178580a711c111e94.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-v3/f2b260eca76347d5982b96c79563999653d141626ca1eadfdb47fc20d645b9d5.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-v3/f317c1c7b57a6ac742de1b97f2f84da6aa7892f8cdd4f22158ba4c700dbe8a81.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-v3/f55e3b684697b9f478393e35202e92b04e51b1ea8b74eaa4a4382a01154b5f5a.yaml create mode 100644 tests/test_litellm/ocr/fixtures/data/reducto-v3/f847dae1a1f6c3777ce3b65421a97f703d67d234593ac188f948142190fdf06d.yaml create mode 100644 tests/test_litellm/ocr/fixtures/migrate.py create mode 100644 tests/test_litellm/ocr/fixtures/mistral.py create mode 100644 tests/test_litellm/ocr/fixtures/models.py create mode 100644 tests/test_litellm/ocr/fixtures/record.py create mode 100644 tests/test_litellm/ocr/fixtures/reducto.py create mode 100644 tests/test_litellm/ocr/fixtures/vertex.py create mode 100644 tests/test_litellm/ocr/test_fixture_models.py create mode 100644 tests/test_litellm/ocr/test_fixture_store.py create mode 100644 tests/test_litellm/ocr/test_record_fixtures.py create mode 100644 tests/test_litellm/ocr/test_sdk_parity.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index c2dff805772..fff77acf5f9 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -106,6 +106,7 @@ jobs: tests/test_litellm/endpoints tests/test_litellm/experimental_mcp_client tests/test_litellm/models + tests/route_parity tests/test_litellm/repositories tests/test_litellm/images tests/test_litellm/interactions diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 69728a652bd..801513156a4 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -25,7 +25,7 @@ rand = "0.8" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] } rstest = "0.26.1" serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +serde_json = { version = "1.0", features = ["float_roundtrip"] } sha2 = "0.10" subtle = "2" thiserror = "2.0" diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs index b26a7925e8a..641a019476e 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs @@ -14,7 +14,7 @@ const AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGE const AZURE_DOCUMENT_INTELLIGENCE_API_VERSION: &str = "2024-11-30"; const AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: i64 = 96; -const AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS: &[&str] = &["pages"]; +const AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS: &[&str] = &["pages", "features"]; pub struct AzureAiOcrConfig; pub struct AzureDocumentIntelligenceOcrConfig; @@ -192,6 +192,46 @@ fn normalize_pages_param(pages: &Value) -> Result, Error> { } } +fn feature_token_is_valid(token: &str) -> bool { + let Some((first, rest)) = token.as_bytes().split_first() else { + return false; + }; + first.is_ascii_alphabetic() && rest.iter().all(u8::is_ascii_alphanumeric) +} + +fn invalid_features_error(features: &Value) -> Error { + Error::InvalidRequest(format!( + "Invalid `features` for Azure Document Intelligence: {features:?}. Expected a list of feature names or a comma-separated string like 'keyValuePairs' or 'keyValuePairs,languages'." + )) +} + +fn normalize_features_param(features: &Value) -> Result, Error> { + let normalized = match features { + Value::String(value) => value + .split(',') + .map(str::trim) + .collect::>() + .join(","), + Value::Array(values) if values.is_empty() => return Ok(None), + Value::Array(values) => values + .iter() + .map(Value::as_str) + .collect::>>() + .ok_or_else(|| invalid_features_error(features))? + .into_iter() + .map(str::trim) + .collect::>() + .join(","), + _ => return Err(invalid_features_error(features)), + }; + + if normalized.split(',').all(feature_token_is_valid) { + Ok(Some(normalized)) + } else { + Err(invalid_features_error(features)) + } +} + pub fn complete_document_intelligence_url( api_base: Option<&str>, model: &str, @@ -213,6 +253,13 @@ pub fn complete_document_intelligence_url( url.push_str(&normalized); } + if let Some(features) = optional_params.get("features") + && let Some(normalized) = normalize_features_param(features)? + { + url.push_str("&features="); + url.push_str(&normalized); + } + Ok(url) } @@ -475,6 +522,103 @@ mod tests { ); } + #[test] + fn document_intelligence_url_normalizes_features() { + let params = serde_json::Map::from_iter([( + "features".to_string(), + json!("keyValuePairs, languages"), + )]); + let url = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect("url builds"); + + assert_eq!( + url, + "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&features=keyValuePairs,languages" + ); + } + + #[test] + fn document_intelligence_url_combines_pages_and_feature_list() { + let params = serde_json::Map::from_iter([ + ("pages".to_string(), json!([0, 1, 2])), + ( + "features".to_string(), + json!([" keyValuePairs ", "languages"]), + ), + ]); + let url = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect("url builds"); + + assert_eq!( + url, + "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&pages=1,2,3&features=keyValuePairs,languages" + ); + } + + #[test] + fn document_intelligence_url_omits_empty_feature_list() { + let params = serde_json::Map::from_iter([("features".to_string(), json!([]))]); + let url = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect("url builds"); + + assert_eq!( + url, + "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30" + ); + } + + #[test] + fn document_intelligence_url_rejects_invalid_features() { + for features in [ + json!("keyValuePairs&pages=9"), + json!(""), + json!(["keyValuePairs", 1]), + json!({"feature": "keyValuePairs"}), + ] { + let params = serde_json::Map::from_iter([("features".to_string(), features.clone())]); + let error = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect_err("invalid features must fail"); + + assert!( + matches!(error, Error::InvalidRequest(message) if message.contains("Invalid `features`")), + "features={features:?}" + ); + } + } + + #[test] + fn document_intelligence_maps_features() { + let params = Map::from_iter([ + ("features".to_string(), json!(["keyValuePairs"])), + ("unsupported".to_string(), json!(true)), + ]); + + assert_eq!( + AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG.map_ocr_params(¶ms), + Map::from_iter([("features".to_string(), json!(["keyValuePairs"]))]) + ); + } + #[test] fn document_intelligence_request_uses_base64_source_for_data_uri() { let body = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 3e9bf9a3bbf..9eaffcab712 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -31,6 +31,16 @@ pub(crate) fn core_error_to_pyerr(err: Error) -> PyErr { } } +pub(crate) fn ocr_error_to_pyerr(err: Error) -> PyErr { + match err { + Error::MissingField("document_url" | "image_url") => { + PyValueError::new_err("Document URL is required") + } + Error::Http { status, body } => RustUpstreamError::new_err((status, body)), + other => core_error_to_pyerr(other), + } +} + /// Map a core error for a route whose host keeps a Python implementation. /// /// Only an explicit capability decline permits the host to try Python. Every @@ -61,6 +71,29 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { mod tests { use super::*; + #[test] + fn ocr_errors_preserve_python_validation_and_provider_details() { + Python::initialize(); + Python::attach(|py| { + for field in ["document_url", "image_url"] { + let mapped = ocr_error_to_pyerr(Error::MissingField(field)); + assert!(mapped.is_instance_of::(py)); + assert_eq!(mapped.value(py).to_string(), "Document URL is required"); + } + let mapped = ocr_error_to_pyerr(Error::Http { + status: 429, + body: r#"{"message":"rate limited"}"#.to_string(), + }); + assert!(mapped.is_instance_of::(py)); + let args: (u16, String) = mapped + .value(py) + .getattr("args") + .and_then(|args| args.extract()) + .expect("OCR failures retain status and unprefixed provider message"); + assert_eq!(args, (429, r#"{"message":"rate limited"}"#.to_string())); + }); + } + #[test] fn fallback_routes_distinguish_declines_from_upstream_failures() { Python::initialize(); diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr.rs b/litellm-rust/crates/python-bridge/src/routes/ocr.rs index 5588c400972..5cc8804238b 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr.rs @@ -5,7 +5,7 @@ use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr}; use pyo3::prelude::*; use serde_json::Value; -use crate::errors::core_error_to_pyerr; +use crate::errors::ocr_error_to_pyerr; use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty}; fn prepare_ocr( @@ -69,5 +69,5 @@ bridge_route! { timeout_seconds: Option, }, prepare = prepare_ocr, - errors = core_error_to_pyerr, + errors = ocr_error_to_pyerr, } diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index e361f68aad6..450c05ecd52 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -53,12 +53,13 @@ class _PreparedOCRRequest: litellm_logging_obj: LiteLLMLoggingObj -@dataclass +@dataclass(frozen=True, slots=True) class _PreparedRustOCRCall: api_key: str | None api_base: str | None headers: dict[str, object] optional_params: dict[str, object] + request_url: str _RUST_OCR_PROVIDERS: Final = { @@ -210,6 +211,10 @@ def _prepare_ocr_request( def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool: if prepared_request.optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native": return False + if prepared_request.extra_headers is not None and any( + not isinstance(value, str) for value in prepared_request.extra_headers.values() + ): + return False return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS @@ -300,6 +305,7 @@ def _prepare_rust_ocr_call( api_base=rust_api_base, headers=cast(dict[str, object], resolved_headers), optional_params=rust_optional_params, + request_url=resolved_complete_url, ) @@ -322,6 +328,7 @@ def _run_rust_ocr( extra_headers=prepared.headers, optional_params=prepared.optional_params, timeout=prepared_request.effective_timeout, + request_url=prepared.request_url, ) if rust_response is None: return None @@ -347,6 +354,7 @@ async def _run_rust_aocr( extra_headers=prepared.headers, optional_params=prepared.optional_params, timeout=prepared_request.effective_timeout, + request_url=prepared.request_url, ) if rust_response is None: return None diff --git a/litellm/rust_bridge/ocr.py b/litellm/rust_bridge/ocr.py index 401ca3448f9..325dbef3440 100644 --- a/litellm/rust_bridge/ocr.py +++ b/litellm/rust_bridge/ocr.py @@ -23,6 +23,14 @@ rust_ocr_enabled = _configuration.rust_ocr_enabled use_litellm_rust = _configuration.use_litellm_rust +class _OcrProviderError(Exception): + def __init__(self, status_code: int, message: str, request_url: str | None) -> None: + super().__init__(message) + self.status_code: Final = status_code + request: Final = httpx.Request("POST", request_url) if request_url is not None else None + self.response: Final = httpx.Response(status_code=status_code, request=request) + + class RustOcr(Protocol): def __call__( self, @@ -84,6 +92,7 @@ def ocr( extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout: float | httpx.Timeout | None, + request_url: str | None = None, ) -> dict[str, object] | None: rust_ocr: Final = load_rust_ocr() native_call: Final = ( @@ -105,7 +114,12 @@ def ocr( fallback=lambda: None, adapt=identity, mode=FallbackMode.PYTHON, - context=BridgeErrorContext(route="ocr", provider=custom_llm_provider or "", model=model), + context=BridgeErrorContext( + route="ocr", + provider=custom_llm_provider or "", + model=model, + upstream_error=lambda status, message: _OcrProviderError(status, message, request_url), + ), ).value @@ -119,6 +133,7 @@ async def aocr( extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout: float | httpx.Timeout | None, + request_url: str | None = None, ) -> dict[str, object] | None: rust_aocr: Final = load_rust_aocr() native_call: Final = ( @@ -141,6 +156,11 @@ async def aocr( fallback=async_none, adapt=identity, mode=FallbackMode.PYTHON, - context=BridgeErrorContext(route="ocr", provider=custom_llm_provider or "", model=model), + context=BridgeErrorContext( + route="ocr", + provider=custom_llm_provider or "", + model=model, + upstream_error=lambda status, message: _OcrProviderError(status, message, request_url), + ), ) ).value diff --git a/litellm/rust_bridge/runtime.py b/litellm/rust_bridge/runtime.py index e27099d4d22..b04447dee3d 100644 --- a/litellm/rust_bridge/runtime.py +++ b/litellm/rust_bridge/runtime.py @@ -55,6 +55,7 @@ class BridgeErrorContext: route: str provider: str model: str + upstream_error: Callable[[int, str], Exception] | None = None def execution_headers(source: CoreEngine) -> dict[str, str]: @@ -221,11 +222,15 @@ def _raise_upstream(error: BaseException, context: BridgeErrorContext) -> NoRetu message_value: Final = args[1] if len(args) > 1 else str(error) status: Final = status_value if isinstance(status_value, int) else 0 message: Final = message_value if isinstance(message_value, str) else str(message_value) - api_error: Final = APIError( - status_code=status or 500, - message=f"litellm rust {context.route}: {message}", - llm_provider=context.provider, - model=context.model, + api_error: Final = ( + context.upstream_error(status or 500, message) + if context.upstream_error is not None + else APIError( + status_code=status or 500, + message=f"litellm rust {context.route}: {message}", + llm_provider=context.provider, + model=context.model, + ) ) api_error.headers = execution_headers( # pyright: ignore[reportAttributeAccessIssue] # proxy reads exception headers CoreEngine.RUST diff --git a/pyproject.toml b/pyproject.toml index 60162544612..596dbb64c8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -174,6 +174,8 @@ litellm-proxy = "litellm.proxy.client.cli:cli" [dependency-groups] dev = [ "diff-cover==9.7.2", + "hypothesis==6.165.10", + "reportlab==5.0.1", "basedpyright==1.39.7", "keyring==25.7.0", "pytest==9.0.3", diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index f5a2fb4b14c..9103d913c36 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -172,6 +172,7 @@ pylint: >=3.3.9 # GPLv2 license langchain-mcp-adapters: >=0.2.1 # MIT License langgraph: >=1.0.10 # MIT License langgraph-prebuilt: >=1.0.8 # MIT License - https://github.com/langchain-ai/langgraph/blob/main/LICENSE +hypothesis: >=6.165.10 # MPL 2.0 license pytest-rerunfailures: >=15.1 # MPL 2.0 license pytest-recording: >=0.13.4 # MIT license expression: >=5.6.0 # MIT License - https://github.com/cognitedata/Expression/blob/main/LICENSE diff --git a/tests/e2e/provider_edge.py b/tests/e2e/provider_edge.py index ceb695ffcd6..e70b50257b1 100644 --- a/tests/e2e/provider_edge.py +++ b/tests/e2e/provider_edge.py @@ -94,6 +94,11 @@ from fixture_mode import ( current_test_key, parse_fixture_mode, ) +from tests.provider_record_replay.http import ( + dropped_request_headers, + dropped_response_headers, + is_streaming_response, +) EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( { @@ -104,29 +109,6 @@ EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( REPLAY_MISS_STATUS: Final = 599 -_HOP_BY_HOP_HEADERS: Final[frozenset[str]] = frozenset( - { - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailers", - "transfer-encoding", - "upgrade", - } -) -_REQUEST_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | { - "host", - "content-length", - "accept-encoding", -} -_RESPONSE_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | { - "content-encoding", - "content-length", - "set-cookie", -} - _JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) @@ -566,9 +548,8 @@ def _filtered_response_headers(headers: Mapping[str, str]) -> dict[str, str]: """What the edge stores and serves: the provider's headers minus hop-by-hop and volatile entries. Framing headers are in that set, so a stored header can never contradict the framing the edge chooses when it serves the response.""" - return { - name: value for name, value in headers.items() if name not in _RESPONSE_DROPPED_HEADERS - } + excluded: Final = dropped_response_headers(headers.items()) + return {name: value for name, value in headers.items() if name.lower() not in excluded} def _network_error_response(message: str) -> RecordedHttpResponse: @@ -610,7 +591,7 @@ def _is_streamed(headers: Mapping[str, str]) -> bool: move nearly every recording to the streamed shape for no gain. The content type is the header that says "consume this as it arrives", and it is already how the harness defines streaming everywhere else.""" - return "text/event-stream" in _header_value(headers, "content-type").lower() + return is_streaming_response(_header_value(headers, "content-type")) def _upstream_url(upstream_base: str, upstream_path: str, query: str) -> str: @@ -698,8 +679,9 @@ def _handle_record( timeout: float, ) -> EdgeOutcome: test_key: Final = current_test_key() + excluded: Final = dropped_request_headers(headers.items()) forwarded: Final = { - name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS + name: value for name, value in headers.items() if name.lower() not in excluded } head: Final = forward_stream(method, url, headers=forwarded, body=body, timeout=timeout) match head: diff --git a/tests/provider_record_replay/__init__.py b/tests/provider_record_replay/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/provider_record_replay/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/provider_record_replay/http.py b/tests/provider_record_replay/http.py new file mode 100644 index 00000000000..c164e3c6549 --- /dev/null +++ b/tests/provider_record_replay/http.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from collections.abc import Iterable +from typing import Final + +HOP_BY_HOP_HEADERS: Final[frozenset[str]] = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "trailers", + "transfer-encoding", + "upgrade", + } +) + +REQUEST_DROPPED_HEADERS: Final[frozenset[str]] = HOP_BY_HOP_HEADERS | { + "host", + "content-length", + "accept-encoding", +} + +RESPONSE_DROPPED_HEADERS: Final[frozenset[str]] = HOP_BY_HOP_HEADERS | { + "content-encoding", + "content-length", + "set-cookie", +} + + +def connection_header_names(headers: Iterable[tuple[str, str]]) -> frozenset[str]: + return frozenset( + token.strip().lower() + for name, value in headers + if name.lower() == "connection" + for token in value.split(",") + if token.strip() + ) + + +def dropped_request_headers(headers: Iterable[tuple[str, str]]) -> frozenset[str]: + materialized: Final = tuple(headers) + return REQUEST_DROPPED_HEADERS | connection_header_names(materialized) + + +def dropped_response_headers(headers: Iterable[tuple[str, str]]) -> frozenset[str]: + materialized: Final = tuple(headers) + return RESPONSE_DROPPED_HEADERS | connection_header_names(materialized) + + +def is_streaming_response(content_type: str) -> bool: + return "text/event-stream" in content_type.lower() diff --git a/tests/route_parity/README.md b/tests/route_parity/README.md new file mode 100644 index 00000000000..468dc531a99 --- /dev/null +++ b/tests/route_parity/README.md @@ -0,0 +1,91 @@ +# Implementation parity testing through the SDK interface + +> Given the same SDK call and identical provider behavior, do two implementations expose the same SDK contract? + +## What the harness compares + +- A fixture contains a LiteLLM SDK input and a recorded upstream provider response +- The same LiteLLM input is transformed by isolated baseline and candidate implementations +- The resulting provider requests must match in method, path, headers, and body, excluding runtime-specific HTTP metadata +- The recorded provider response is then replayed unchanged to both workers +- The harness compares the values returned through the Python SDK interface +- Non-streaming responses are compared directly, including their concrete return type and public model fields +- Streaming responses are consumed and compared chunk by chunk, including wrapper type, chunk type and order, termination, and public exception behavior +- Failed SDK calls are compared by exception class, stable message, status, code, model, provider, and parameter fields +- Traceback paths and line numbers are excluded because they are runtime-specific +- Route-specific comparators and chunk normalizers handle differences in each public SDK contract + +## Process isolation + +- SDK object and stream parity runs both implementations sequentially in the same process so tests can retain returned objects +- Every test saves and restores the original bridge state +- A small subprocess smoke test verifies environment-based startup configuration and detects fallback to the Python HTTP implementation + +## Streaming execution + +The invocation callback passed to `run_in_process` must consume the stream before returning its `StreamOutcome`. +Use `consume_sync_stream` inside that callback, or await `consume_async_stream` inside the callback passed to +`run_in_process_async`. Provider requests are collected only after the callback completes. Streaming is explicit: +an iterable return value alone does not select stream consumption + +The consumers retain the wrapper type, iteration capabilities, chunk types and order, and any partial output before +an error. Errors retain their creation or iteration phase and the full public `SDKError` fields, with traceback text +removed. `capture_sync_stream` and `capture_async_stream` consume through the same helpers and then serialize the +outcome for subprocess reports. A serialization failure raises as a harness failure rather than becoming an SDK error + +Response models and stream chunks share a recursive comparator. It compares concrete model, container, and scalar +types, public fields and extras, and exact values while ignoring Pydantic private attributes at every nesting level. +An API may supply an explicit chunk normalizer for its public contract + +Shared tests exercise a local SSE provider through recording, VCR cassette storage, replay, and typed event comparison +in sync and async modes. They cover fragmented events, split UTF-8 characters, CRLF framing, coalesced events, and +application errors within a normally completed HTTP stream. HTTP byte boundaries and decoded SDK event boundaries +are checked separately + +OCR remains the only integrated LiteLLM route. These tests validate shared streaming machinery, not another route's +SDK parity. Connection interruption, early cancellation, and lifecycle timeout enforcement remain outside this coverage + +## Hypothesis and property-based testing + +- Hypothesis is a Python library for property-based testing +- Example-based tests use inputs selected by the test author +- Property-based tests define strategies for valid inputs and properties that must hold for every generated example +- Hypothesis generates combinations from those strategies and normally shrinks a failing example to a smaller reproducible case +- In this harness, Hypothesis is used only during fixture generation to expand the LiteLLM input corpus +- Each API owns the strategies that vary its supported inputs +- Fixture generation is deterministic, and each generated input is recorded with the raw provider response it received +- The parity tests use committed fixtures and do not call the provider or generate new Hypothesis examples +- Provider responses are replayed unchanged, so the parity test does not fuzz or validate provider behavior +- Because Hypothesis does not run the parity assertion directly, parity failures are not automatically shrunk + +## API-owned fixtures + +The shared package owns recording, replay, persistence, execution, comparison, and route-neutral media constructors. +Each API package owns its input models, explicit strategies, provider targets, route-specific assets, fixture directory, +and regeneration command. See the API package documentation for its configured contracts and recording command + +## VCR cassettes + +Fixtures use VCR's YAML `version: 1` format with ordered request/response `interactions`. VCR handles text and binary +body serialization. Each cassette also contains `recorded_at`, `ttl_seconds: 0` (committed fixtures never expire), and +`x-litellm` metadata holding the SDK input and request provenance. Streaming responses carry +`x-litellm-chunk-lengths` so local replay preserves the original byte boundaries + +The recording server captures requests before forwarding their responses. Saved requests use the stable +`http://parity-provider.invalid` origin and strip authentication headers and credential query parameters. The upstream +request keeps its credentials. Provider response bytes and non-success statuses are preserved + +Standard VCR can load these files and replay their interactions. Parity tests keep using the local HTTP server because +Rust HTTP calls do not pass through VCR's Python patches. The harness still compares the two implementations' requests +against each other; the saved request is available for inspection and VCR playback, not a new parity assertion + +Refresh parity cassettes through the API's recording command. Generic VCR writers do not preserve the SDK metadata + +Legacy JSON fixtures remain readable. Migrated cassettes mark reconstructed requests as `python_replay`; fresh +recordings use `recorded`. The metadata extensions follow the filesystem cassette layout proposed in +[PR #39338](https://github.com/BerriAI/litellm/pull/39338), without depending on its unmerged persistence backend + +## References + +- [Hypothesis documentation](https://hypothesis.readthedocs.io/en/latest/) +- [Hypothesis documentation source](https://github.com/HypothesisWorks/hypothesis/tree/master/hypothesis/docs) diff --git a/tests/route_parity/__init__.py b/tests/route_parity/__init__.py new file mode 100644 index 00000000000..df2a40db636 --- /dev/null +++ b/tests/route_parity/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.register_assert_rewrite("tests.route_parity.compare") diff --git a/tests/route_parity/compare.py b/tests/route_parity/compare.py new file mode 100644 index 00000000000..56e539c1258 --- /dev/null +++ b/tests/route_parity/compare.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Final, cast + +from pydantic import BaseModel + +from tests.route_parity.models import CapturedRequest, Execution + + +def validate_harness(baseline: Execution, candidate: Execution, baseline_user_agent: str) -> None: + for request in baseline.requests: + if request.user_agent != baseline_user_agent: + raise AssertionError( + f"baseline provider request did not carry sentinel user-agent {baseline_user_agent!r}: " + f"{request.user_agent!r}" + ) + for request in candidate.requests: + if request.user_agent == baseline_user_agent: + raise AssertionError("candidate route fell back to the baseline HTTP implementation") + + +def _request_after_transformation(request: CapturedRequest) -> CapturedRequest: + return request.model_copy(update={"user_agent": None}) + + +def assert_request_parity(baseline: tuple[CapturedRequest, ...], candidate: tuple[CapturedRequest, ...]) -> None: + baseline_requests: Final = tuple(_request_after_transformation(request) for request in baseline) + candidate_requests: Final = tuple(_request_after_transformation(request) for request in candidate) + assert_value_parity(baseline_requests, candidate_requests) + + +def _public_model_values(model: BaseModel) -> dict[str, object]: + fields: Final = (*type(model).model_fields, *type(model).model_computed_fields) + extras: Final = cast(Mapping[str, object], model.model_extra or {}) + return { + **{name: cast(object, getattr(model, name)) for name in fields if not name.startswith("_")}, + **{name: value for name, value in extras.items() if not name.startswith("_")}, + } + + +def assert_model_parity(baseline: BaseModel, candidate: BaseModel) -> None: + assert_value_parity(baseline, candidate) + + +def assert_value_parity(baseline: object, candidate: object, *, path: str = "$") -> None: + assert type(baseline) is type(candidate), f"type mismatch at {path}: {type(baseline)} != {type(candidate)}" + if isinstance(baseline, BaseModel) and isinstance(candidate, BaseModel): + assert_value_parity(_public_model_values(baseline), _public_model_values(candidate), path=path) + return + if isinstance(baseline, Mapping) and isinstance(candidate, Mapping): + baseline_mapping: Final = cast(Mapping[object, object], baseline) + candidate_mapping: Final = cast(Mapping[object, object], candidate) + assert frozenset((type(key), key) for key in baseline_mapping) == frozenset( + (type(key), key) for key in candidate_mapping + ), f"mapping keys differ at {path}" + for key in baseline_mapping: + assert_value_parity(baseline_mapping[key], candidate_mapping[key], path=f"{path}.{key}") + return + if ( + isinstance(baseline, Sequence) + and not isinstance(baseline, (str, bytes)) + and isinstance(candidate, Sequence) + and not isinstance(candidate, (str, bytes)) + ): + baseline_sequence: Final = cast(Sequence[object], baseline) + candidate_sequence: Final = cast(Sequence[object], candidate) + assert len(baseline_sequence) == len(candidate_sequence), f"sequence lengths differ at {path}" + for index, (baseline_item, candidate_item) in enumerate( + zip(baseline_sequence, candidate_sequence, strict=True) + ): + assert_value_parity(baseline_item, candidate_item, path=f"{path}[{index}]") + return + assert baseline == candidate, f"value mismatch at {path}: {baseline!r} != {candidate!r}" + + +def assert_parity(baseline: Execution, candidate: Execution, baseline_user_agent: str) -> None: + validate_harness(baseline, candidate, baseline_user_agent) + assert_request_parity(baseline.requests, candidate.requests) + assert_value_parity(baseline.report, candidate.report) diff --git a/tests/route_parity/fixture_models.py b/tests/route_parity/fixture_models.py new file mode 100644 index 00000000000..f1ed9dc3fe9 --- /dev/null +++ b/tests/route_parity/fixture_models.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import ClassVar, Final, Generic, Literal, TypeVar, cast + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator + +from tests.route_parity.recorded_http import RecordedResponse + +JsonObject = dict[str, JsonValue] + + +class FixtureModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True, serialize_by_alias=True) + + +class SdkInputBase(FixtureModel): + fixture_only_fields: ClassVar[tuple[str, ...]] = () + + def as_sdk_kwargs(self) -> dict[str, object]: + return cast( + dict[str, object], + self.model_dump( + mode="python", + exclude_unset=True, + exclude=set(self.fixture_only_fields), + ), + ) + + def canonical_input(self) -> dict[str, object]: + dumped: Final = cast(dict[str, object], self.model_dump(mode="json", exclude_unset=True)) + fixture_fields: Final = {field: getattr(self, field) for field in self.fixture_only_fields} + return {**fixture_fields, **dumped} + + +class JsonSchemaDefinition(FixtureModel): + name: str + description: str | None = None + schema_definition: JsonObject = Field(alias="schema") + strict: bool = False + + +class JsonSchemaResponseFormat(FixtureModel): + type: Literal["json_schema"] + json_schema: JsonSchemaDefinition + + +InputT = TypeVar("InputT", bound=SdkInputBase) + + +class ParityCase(FixtureModel, Generic[InputT]): + litellm_input: InputT + provider_responses: tuple[RecordedResponse, ...] + + @model_validator(mode="before") + @classmethod + def load_legacy_single_response(cls, value: object) -> object: + if not isinstance(value, Mapping): + return value + migrated: Final = dict(cast(Mapping[str, object], value)) + provider_response: Final = migrated.pop("provider_response", None) + if "provider_responses" not in migrated and provider_response is not None: + migrated["provider_responses"] = (provider_response,) + return migrated diff --git a/tests/route_parity/fixtures/__init__.py b/tests/route_parity/fixtures/__init__.py new file mode 100644 index 00000000000..9d48db4f9f8 --- /dev/null +++ b/tests/route_parity/fixtures/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/route_parity/fixtures/cassette.py b/tests/route_parity/fixtures/cassette.py new file mode 100644 index 00000000000..6ab4210e031 --- /dev/null +++ b/tests/route_parity/fixtures/cassette.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +from collections.abc import Mapping +from datetime import datetime +from itertools import accumulate +from typing import Final, Literal + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, TypeAdapter +from vcr.serialize import serialize +from vcr.serializers import yamlserializer + +from tests.route_parity.fixtures.recording import RecordedInteraction +from tests.route_parity.recorded_http import ( + HttpHeader, + RecordedHttpResponse, + RecordedHttpStreamResponse, + RecordedResponse, + RecordedStreamChunk, +) + +_OBJECT: Final = TypeAdapter(dict[str, object]) + + +class _CassetteModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) + + +class _Body(_CassetteModel): + string: str | bytes + + def as_bytes(self) -> bytes: + return self.string.encode("utf-8") if isinstance(self.string, str) else self.string + + +class _Status(_CassetteModel): + code: int + message: str + + +class _Request(_CassetteModel): + method: str + uri: str + body: str | bytes | None + headers: dict[str, tuple[str, ...]] + + +class _Response(_CassetteModel): + status: _Status + headers: dict[str, tuple[str, ...]] + body: _Body + chunk_lengths: tuple[int, ...] | None = Field(default=None, alias="x-litellm-chunk-lengths") + + def recorded_response(self) -> RecordedResponse: + headers: Final = tuple( + HttpHeader(name=name, value=value) for name, values in self.headers.items() for value in values + ) + body: Final = self.body.as_bytes() + if self.chunk_lengths is None: + return RecordedHttpResponse.from_bytes(self.status.code, headers, body) + if any(length < 0 for length in self.chunk_lengths) or sum(self.chunk_lengths) != len(body): + raise ValueError("cassette stream chunk lengths do not match the response body") + offsets: Final = tuple(accumulate(self.chunk_lengths, initial=0)) + return RecordedHttpStreamResponse( + kind="http_stream", + status_code=self.status.code, + headers=headers, + chunks=tuple(RecordedStreamChunk.from_bytes(body[start:end]) for start, end in zip(offsets, offsets[1:])), + ) + + +class _Interaction(_CassetteModel): + request: _Request + response: _Response + + +class _ParityMetadata(_CassetteModel): + schema_version: Literal[1] + request_source: Literal["recorded", "python_replay"] + case: dict[str, object] + + +class ParityCassette(_CassetteModel): + version: Literal[1] + recorded_at: AwareDatetime + ttl_seconds: Literal[0] + interactions: tuple[_Interaction, ...] + parity: _ParityMetadata = Field(alias="x-litellm") + + def case_data(self) -> dict[str, object]: + return { + **self.parity.case, + "provider_responses": tuple(item.response.recorded_response() for item in self.interactions), + } + + +def _response_dict(response: RecordedResponse) -> dict[str, object]: + headers: Final = { + name: [header.value for header in response.headers if header.name == name] + for name in dict.fromkeys(header.name for header in response.headers) + } + chunks: Final = ( + tuple(chunk.data_bytes() for chunk in response.chunks) + if isinstance(response, RecordedHttpStreamResponse) + else None + ) + body: Final = response.body_bytes() if isinstance(response, RecordedHttpResponse) else b"".join(chunks or ()) + return { + "status": {"code": response.status_code, "message": ""}, + "headers": headers, + "body": {"string": body}, + **({"x-litellm-chunk-lengths": list(map(len, chunks))} if chunks is not None else {}), + } + + +def serialize_cassette( + case: Mapping[str, object], + interactions: tuple[RecordedInteraction, ...], + recorded_at: datetime, + request_source: Literal["recorded", "python_replay"], +) -> str: + normalized: Final = _OBJECT.validate_python( + yamlserializer.deserialize( + serialize( + { + "requests": [item.request for item in interactions], + "responses": [_response_dict(item.response) for item in interactions], + }, + yamlserializer, + ) + ) + ) + payload: Final = { + **normalized, + "recorded_at": recorded_at.isoformat(), + "ttl_seconds": 0, + "x-litellm": { + "schema_version": 1, + "request_source": request_source, + "case": {key: value for key, value in case.items() if key != "provider_responses"}, + }, + } + ParityCassette.model_validate(payload).case_data() + return str(yamlserializer.serialize(payload)) + + +def deserialize_cassette(contents: str) -> ParityCassette: + return ParityCassette.model_validate(yamlserializer.deserialize(contents)) diff --git a/tests/route_parity/fixtures/cli.py b/tests/route_parity/fixtures/cli.py new file mode 100644 index 00000000000..b6a2db1c395 --- /dev/null +++ b/tests/route_parity/fixtures/cli.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import argparse +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final, cast + + +@dataclass(frozen=True, slots=True) +class RecordingArgs: + concurrency: int + examples: int + fixture_dir: Path | None + + +def _positive_int(value: str) -> int: + parsed: Final = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be at least 1") + return parsed + + +def parse_recording_args(argv: Sequence[str] | None = None) -> RecordingArgs: + parser: Final = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=_positive_int, default=4) + parser.add_argument("--examples", type=_positive_int, default=4) + parser.add_argument("--fixture-dir", type=Path) + namespace: Final = parser.parse_args(argv) + return RecordingArgs( + concurrency=cast(int, namespace.concurrency), + examples=cast(int, namespace.examples), + fixture_dir=cast(Path | None, namespace.fixture_dir), + ) diff --git a/tests/route_parity/fixtures/inputs.py b/tests/route_parity/fixtures/inputs.py new file mode 100644 index 00000000000..bbca19d0104 --- /dev/null +++ b/tests/route_parity/fixtures/inputs.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import queue +from typing import Final, TypeVar + +from hypothesis import given, settings +from hypothesis.strategies import SearchStrategy + +InputT = TypeVar("InputT") + + +def generate_case_inputs(strategy: SearchStrategy[InputT], examples: int) -> tuple[InputT, ...]: + generated: Final[queue.SimpleQueue[InputT | None]] = queue.SimpleQueue() + + @settings(max_examples=examples, deadline=None, derandomize=True) + @given(case_input=strategy) + def generate_case(case_input: InputT) -> None: + generated.put(case_input) + + generate_case() + generated.put(None) + return tuple(iter(generated.get, None)) diff --git a/tests/route_parity/fixtures/media.py b/tests/route_parity/fixtures/media.py new file mode 100644 index 00000000000..d1ea2901f5d --- /dev/null +++ b/tests/route_parity/fixtures/media.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import base64 +from functools import cache +from io import BytesIO +from typing import Final +from urllib.parse import quote + +from PIL import Image, ImageDraw +from reportlab.graphics.barcode import code128 # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.lib import colors # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.lib.pagesizes import letter # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.lib.utils import ImageReader # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.pdfgen import canvas # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs + + +def dummy_image_url(text: str, font_size: int, width: int = 800, height: int = 300) -> str: + return f"https://dummyjson.com/image/{width}x{height}/ffffff/000000?text={quote(text)}&fontSize={font_size}" + + +_GLYPHS: Final = { + "D": ("11110", "10001", "10001", "10001", "10001", "10001", "11110"), + "O": ("01110", "10001", "10001", "10001", "10001", "10001", "01110"), + "C": ("01111", "10000", "10000", "10000", "10000", "10000", "01111"), + "1": ("00100", "01100", "00100", "00100", "00100", "00100", "01110"), + "2": ("01110", "10001", "00001", "00010", "00100", "01000", "11111"), + "3": ("11110", "00001", "00001", "01110", "00001", "00001", "11110"), +} + + +@cache +def structured_image_bytes() -> bytes: + image: Final = Image.new("RGB", (320, 80), "white") + draw: Final = ImageDraw.Draw(image) + scale: Final = 8 + cursor_x = 24 + for character in "DOC 123": + if character == " ": + cursor_x += scale * 3 + continue + for glyph_y, row in enumerate(_GLYPHS[character]): + for glyph_x, filled in enumerate(row): + if filled == "1": + x = cursor_x + glyph_x * scale + y = 12 + glyph_y * scale + draw.rectangle((x, y, x + scale - 1, y + scale - 1), fill="black") + cursor_x += scale * 6 + output: Final = BytesIO() + image.save(output, format="PNG") + return output.getvalue() + + +@cache +def structured_image_data_uri() -> str: + encoded: Final = base64.b64encode(structured_image_bytes()).decode("ascii") + return f"data:image/png;base64,{encoded}" + + +def _draw_header(pdf: canvas.Canvas, title: str, page_number: int) -> None: + pdf.setFillColor(colors.black) + pdf.setFont("Helvetica", 11) + pdf.drawString(45, 770, "Quarterly Operations Report") + pdf.setFont("Helvetica-Bold", 16) + pdf.drawString(45, 745, title) + pdf.setFont("Helvetica", 9) + pdf.drawString(45, 30, f"Confidential | Page {page_number} of 5") + + +def _draw_body(pdf: canvas.Canvas, page_number: int) -> None: + pdf.setFont("Helvetica", 10) + for line_number in range(1, 9): + pdf.drawString( + 45, + 500 - (line_number * 28), + f"Section {page_number}.{line_number}: Invoice totals, regional revenue, and reconciliation notes.", + ) + + +def _diagram_image(width: int, height: int, accent: tuple[int, int, int]) -> Image.Image: + image: Final = Image.new("RGB", (width, height), (242, 246, 252)) + draw: Final = ImageDraw.Draw(image) + for coordinate in range(0, max(width, height), 40): + draw.line((coordinate, 0, coordinate, height), fill=(32, 32, 32), width=3) + draw.line((0, coordinate, width, coordinate), fill=(32, 32, 32), width=3) + draw.line((0, 0, width, height), fill=accent, width=8) + draw.line((width, 0, 0, height), fill=accent, width=8) + draw.rectangle((width // 4, height // 4, width * 3 // 4, height * 3 // 4), outline=accent, width=6) + return image + + +def _draw_embedded_images(pdf: canvas.Canvas) -> None: + images: Final = ( + (_diagram_image(320, 320, (51, 115, 217)), 455, 655, 70, 70), + (_diagram_image(360, 320, (38, 151, 92)), 455, 565, 70, 62), + (_diagram_image(120, 120, (219, 68, 55)), 455, 500, 45, 45), + ) + for image, x, y, width, height in images: + pdf.drawImage( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + ImageReader(image), x, y, width=width, height=height, mask="auto" + ) + + +def _draw_table_page(pdf: canvas.Canvas) -> None: + columns: Final = (45, 245, 405, 565) + tables: Final = ( + ( + (730, 695, 660, 625), + ( + ("Item", "Quantity", "Amount", 707), + ("Document analysis", "2", "120.00", 672), + ("Document verification", "1", "80.00", 637), + ), + ), + ( + (600, 565, 530, 495), + ( + ("Item continued", "Quantity", "Amount", 577), + ("Fixture validation", "3", "45.00", 542), + ("Provider review", "1", "25.00", 507), + ), + ), + ) + for rows, values in tables: + for x in columns: + pdf.line(x, rows[-1], x, rows[0]) + for y in rows: + pdf.line(45, y, 565, y) + for item, quantity, amount, y in values: + pdf.drawString(55, y, item) + pdf.drawString(255, y, quantity) + pdf.drawString(415, y, amount) + + +def _draw_chart_page(pdf: canvas.Canvas) -> None: + bars: Final = ((70, 70), (170, 115), (270, 90), (370, 130)) + pdf.setFillColor(colors.HexColor("#3373D9")) + for x, height in bars: + pdf.rect(x, 610, 65, height, fill=1, stroke=0) + pdf.setFillColor(colors.black) + for quarter, x in zip(("Q1", "Q2", "Q3", "Q4"), (90, 190, 290, 390), strict=True): + pdf.drawString(x, 590, quarter) + pdf.drawString(45, 550, "Formula: gross margin = (revenue - cost) / revenue") + _draw_embedded_images(pdf) + + +def _draw_metadata_page(pdf: canvas.Canvas) -> None: + pdf.setFont("Helvetica", 12) + pdf.drawString(45, 700, "Invoice Number: INV-2048") + pdf.drawString(45, 675, "Purchase Order: PO-4096") + pdf.setFillColor(colors.HexColor("#F2E65A")) + pdf.rect(40, 555, 500, 24, fill=1, stroke=0) + pdf.setFillColor(colors.black) + pdf.drawString(45, 560, "Highlighted total requiring review") + pdf.drawString(45, 530, "Reviewer comment: verify the highlighted total before approval") + pdf.setFillColor(colors.red) + pdf.drawString(45, 495, "Revised total: 245.00") + pdf.line(45, 501, 150, 501) + pdf.setFillColor(colors.black) + pdf.linkURL( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + "https://example.com/invoices/INV-2048", (45, 575, 300, 590), relative=0 + ) + pdf.highlightAnnotation( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + "Total highlighted for review", + Rect=(40, 555, 540, 579), + QuadPoints=(40, 579, 540, 579, 40, 555, 540, 555), + ) + pdf.textAnnotation( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + "Verify the highlighted total", Rect=(520, 525, 540, 545) + ) + pdf.drawString(45, 575, "https://example.com/invoices/INV-2048") + barcode: Final = code128.Code128("5901234123457", barHeight=70, barWidth=1.2) + barcode.drawOn(pdf, 90, 130) + + +def _draw_signature_page(pdf: canvas.Canvas) -> None: + pdf.saveState() + pdf.setFillColor(colors.lightgrey) + pdf.setFont("Helvetica-Bold", 54) + pdf.translate(110, 390) + pdf.rotate(25) + pdf.drawString(0, 0, "DRAFT") + pdf.restoreState() + pdf.setFillColor(colors.black) + pdf.setFont("Helvetica", 12) + pdf.drawString(45, 635, "Approved by: Jordan Lee") + pdf.line(45, 610, 310, 610) + pdf.bezier(55, 595, 75, 625, 112, 602, 155, 600) + pdf.drawString(45, 580, "Signature") + + +def _draw_appendix_page(pdf: canvas.Canvas) -> None: + pdf.setFont("Helvetica-Bold", 14) + pdf.drawString(45, 700, "1. Scope") + pdf.drawString(45, 650, "2. Findings") + pdf.drawString(45, 600, "3. Recommendations") + + +@cache +def structured_pdf_bytes() -> bytes: + output: Final = BytesIO() + pdf: Final = canvas.Canvas(output, pagesize=letter, pageCompression=0, invariant=1) + pdf.setTitle("Quarterly Operations Report") + pdf.setAuthor("LiteLLM parity fixture generator") + pdf.setSubject("Semantic document coverage for tables, figures, annotations, and metadata") + pdf.setKeywords("document, invoice, table, figure, annotation") + pages: Final = ( + ("Invoice Summary and Line Items", _draw_table_page), + ("Revenue Chart and Formula Review", _draw_chart_page), + ("Key Values, Link, Highlight, and Comment", _draw_metadata_page), + ("Approval Signature and Watermark", _draw_signature_page), + ("Appendix with Section Boundaries", _draw_appendix_page), + ) + for page_number, (title, draw_page) in enumerate(pages, start=1): + _draw_header(pdf, title, page_number) + draw_page(pdf) + _draw_body(pdf, page_number) + pdf.showPage() + pdf.save() + return output.getvalue() + + +@cache +def structured_pdf_data_uri() -> str: + encoded: Final = base64.b64encode(structured_pdf_bytes()).decode("ascii") + return f"data:application/pdf;base64,{encoded}" diff --git a/tests/route_parity/fixtures/pipeline.py b/tests/route_parity/fixtures/pipeline.py new file mode 100644 index 00000000000..288fbeba7fa --- /dev/null +++ b/tests/route_parity/fixtures/pipeline.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import logging +from concurrent.futures import Future, ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Final, Generic, Literal, Protocol, TypeVar + +from hypothesis.strategies import SearchStrategy +from pydantic import BaseModel + +from tests.route_parity.fixtures.inputs import generate_case_inputs +from tests.route_parity.fixtures.recording import UpstreamEndpoint, record_upstream_interactions +from tests.route_parity.fixtures.store import ( + FixtureInput, + canonical_json, + fixture_cache_key, + fixture_id, + fixture_path, + load_fixture, + save_fixture, +) + +LOGGER: Final = logging.getLogger(__name__) +InputT = TypeVar("InputT", bound=FixtureInput) +InputT_contra = TypeVar("InputT_contra", bound=FixtureInput, contravariant=True) +CaseT = TypeVar("CaseT", bound=BaseModel) + + +class RecordingInvocation(Protocol[InputT_contra]): + def execute(self, provider_url: str, case_input: InputT_contra) -> None: ... + + +@dataclass(frozen=True, slots=True) +class RecordingTarget(Generic[InputT]): + name: str + upstream: UpstreamEndpoint + strategy: SearchStrategy[InputT] + invocation: RecordingInvocation[InputT] = field(repr=False) + required_inputs: tuple[InputT, ...] = () + + +@dataclass(frozen=True, slots=True) +class RecordingJob(Generic[InputT]): + target_name: str + directory: Path + upstream: UpstreamEndpoint + case_input: InputT + invocation: RecordingInvocation[InputT] = field(repr=False) + + @property + def case_id(self) -> str: + return fixture_id(self.case_input, self.target_name) + + +@dataclass(frozen=True, slots=True) +class RecordedFixture: + target_name: str + case_id: str + path: Path + kind: Literal["recorded"] = field(default="recorded", init=False) + + +@dataclass(frozen=True, slots=True) +class CachedFixture: + target_name: str + case_id: str + path: Path + kind: Literal["cached"] = field(default="cached", init=False) + + +@dataclass(frozen=True, slots=True) +class FailedFixture: + target_name: str + case_id: str + error: Exception = field(repr=False) + kind: Literal["failed"] = field(default="failed", init=False) + + +RecordingOutcome = RecordedFixture | CachedFixture | FailedFixture + + +@dataclass(frozen=True, slots=True) +class RecordingSummary: + recorded: tuple[RecordedFixture, ...] + cached: tuple[CachedFixture, ...] + failed: tuple[FailedFixture, ...] + + @property + def exit_code(self) -> int: + return 1 if self.failed else 0 + + +def _unique_inputs(target: RecordingTarget[InputT], examples: int) -> tuple[InputT, ...]: + generated_inputs: Final = generate_case_inputs(target.strategy, examples) + case_inputs: Final = (*target.required_inputs, *generated_inputs) + return tuple({canonical_json(fixture_cache_key(case_input)): case_input for case_input in case_inputs}.values()) + + +def build_recording_jobs( + targets: tuple[RecordingTarget[InputT], ...], + root: Path, + examples: int, +) -> tuple[RecordingJob[InputT], ...]: + if examples < 1: + raise ValueError("examples must be at least 1") + return tuple( + RecordingJob( + target_name=target.name, + directory=root / target.name, + upstream=target.upstream, + case_input=case_input, + invocation=target.invocation, + ) + for target in targets + for case_input in _unique_inputs(target, examples) + ) + + +def _record_job(job: RecordingJob[InputT], case_type: type[CaseT]) -> RecordedFixture | CachedFixture: + cached: Final = load_fixture(job.directory, job.case_input, case_type) + if cached is not None: + path: Final = fixture_path(job.directory, job.case_input) + return CachedFixture( + target_name=job.target_name, + case_id=job.case_id, + path=path if path.is_file() else path.with_suffix(".json"), + ) + interactions: Final = record_upstream_interactions( + job.upstream, + job.case_input, + job.invocation.execute, + ) + case: Final = case_type.model_validate( + { + "litellm_input": job.case_input, + "provider_responses": tuple(item.response for item in interactions), + } + ) + saved_path: Final = save_fixture(job.directory, job.case_input, case, interactions) + return RecordedFixture(target_name=job.target_name, case_id=job.case_id, path=saved_path) + + +def _completed_outcome( + completed: int, + total: int, + job: RecordingJob[InputT], + future: Future[RecordedFixture | CachedFixture], +) -> RecordingOutcome: + try: + outcome: Final = future.result() + except Exception as error: + failed: Final = FailedFixture(target_name=job.target_name, case_id=job.case_id, error=error) + LOGGER.error( + "[%d/%d] failed %s %s: %s", + completed, + total, + failed.target_name, + failed.case_id, + type(error).__name__, + ) + return failed + LOGGER.info("[%d/%d] %s %s %s", completed, total, outcome.kind, outcome.target_name, outcome.case_id) + return outcome + + +def record_fixtures( + targets: tuple[RecordingTarget[InputT], ...], + root: Path, + examples: int, + concurrency: int, + case_type: type[CaseT], +) -> RecordingSummary: + if concurrency < 1: + raise ValueError("concurrency must be at least 1") + jobs: Final = build_recording_jobs(targets, root, examples) + total: Final = len(jobs) + LOGGER.info("Recording %d fixtures across %d targets with concurrency %d", total, len(targets), concurrency) + with ThreadPoolExecutor(max_workers=concurrency) as executor: + future_jobs: Final = MappingProxyType({executor.submit(_record_job, job, case_type): job for job in jobs}) + outcomes: Final = tuple( + _completed_outcome(completed, total, future_jobs[future], future) + for completed, future in enumerate(as_completed(future_jobs), start=1) + ) + summary: Final = RecordingSummary( + recorded=tuple(outcome for outcome in outcomes if isinstance(outcome, RecordedFixture)), + cached=tuple(outcome for outcome in outcomes if isinstance(outcome, CachedFixture)), + failed=tuple(outcome for outcome in outcomes if isinstance(outcome, FailedFixture)), + ) + LOGGER.info( + "Finished %d fixtures: %d recorded, %d cached, %d failed", + total, + len(summary.recorded), + len(summary.cached), + len(summary.failed), + ) + return summary diff --git a/tests/route_parity/fixtures/pytest_support.py b/tests/route_parity/fixtures/pytest_support.py new file mode 100644 index 00000000000..c4582ff62a4 --- /dev/null +++ b/tests/route_parity/fixtures/pytest_support.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import os +from collections.abc import Callable +from pathlib import Path +from typing import Final, TypeVar + +import pytest +from pydantic import BaseModel, ValidationError + +from tests.route_parity.fixtures.store import recorded_fixtures + +CaseT = TypeVar("CaseT", bound=BaseModel) + + +def parametrize_recorded_fixtures( + metafunc: pytest.Metafunc, + *, + fixture_name: str, + case_type: type[CaseT], + env_var: str, + default_directory: Path, + regeneration_command: str, + id_builder: Callable[[CaseT], str], + marks_builder: Callable[[CaseT], tuple[pytest.MarkDecorator, ...]] | None = None, +) -> None: + if fixture_name not in metafunc.fixturenames: + return + configured: Final = os.environ.get(env_var) + if configured == "": + raise pytest.UsageError(f"{env_var} is set but empty") + directory: Final = Path(configured).expanduser() if configured is not None else default_directory + try: + fixtures: Final = recorded_fixtures(directory, case_type) + except (ValidationError, ValueError) as error: + raise pytest.UsageError( + f"Invalid parity fixture bundle at {directory}. " + "Each fixture must use the current versioned envelope. " + f"Record fresh fixtures in an empty directory with: `{regeneration_command}`. " + f"Validation details: {error}" + ) from error + if fixtures: + metafunc.parametrize( + fixture_name, + tuple( + pytest.param( + fixture, + id=id_builder(fixture), + marks=marks_builder(fixture) if marks_builder is not None else (), + ) + for fixture in fixtures + ), + ) + return + if configured is not None: + raise pytest.UsageError(f"no recorded fixtures in {directory}") + metafunc.parametrize( + fixture_name, + ( + pytest.param( + None, + marks=pytest.mark.skip(reason=f"no recorded fixtures in {directory}"), + id="no-recorded-fixtures", + ), + ), + ) diff --git a/tests/route_parity/fixtures/recording.py b/tests/route_parity/fixtures/recording.py new file mode 100644 index 00000000000..9908a89ae91 --- /dev/null +++ b/tests/route_parity/fixtures/recording.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +import queue +import threading +from collections.abc import Callable, Generator, Iterable +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final, TypeVar, cast +from urllib.parse import urlsplit, urlunsplit + +import httpx +from vcr.filters import remove_query_parameters +from vcr.request import Request + +from tests.provider_record_replay.http import ( + dropped_request_headers, + dropped_response_headers, + is_streaming_response, +) +from tests.route_parity.recorded_http import ( + HttpHeader, + RecordedHttpResponse, + RecordedHttpStreamResponse, + RecordedResponse, + RecordedStreamChunk, +) + +_PARITY_PROVIDER_HOST: Final = "parity-provider.invalid" +_SECRET_HEADERS: Final = frozenset( + { + "authorization", + "proxy-authorization", + "cookie", + "x-api-key", + "api-key", + "anthropic-api-key", + "openai-api-key", + "azure-api-key", + "x-goog-api-key", + "ocp-apim-subscription-key", + "x-amz-security-token", + } +) + +InputT = TypeVar("InputT") + + +@dataclass(frozen=True, slots=True) +class UpstreamEndpoint: + base_url: str + + +@dataclass(frozen=True, slots=True) +class RecordedInteraction: + request: Request + response: RecordedResponse + + +def _end_to_end_headers(headers: httpx.Headers) -> tuple[HttpHeader, ...]: + decoded: Final = tuple((name.decode("ascii"), value.decode("latin-1")) for name, value in headers.raw) + excluded: Final = dropped_response_headers(decoded) + return tuple( + HttpHeader(name=name, value=_normalized_response_header(name, value)) + for name, value in decoded + if name.lower() not in excluded + ) + + +def _normalized_response_header(name: str, value: str) -> str: + if name.lower() not in {"location", "operation-location"}: + return value + parsed: Final = urlsplit(value) + if not parsed.netloc: + return value + return urlunsplit(("http", _PARITY_PROVIDER_HOST, parsed.path, parsed.query, parsed.fragment)) + + +def local_response_header(name: str, value: str, provider_url: str) -> str: + if name.lower() not in {"location", "operation-location"}: + return value + parsed: Final = urlsplit(value) + if parsed.hostname != _PARITY_PROVIDER_HOST: + return value + return f"{provider_url}{parsed.path}{'?' + parsed.query if parsed.query else ''}" + + +class _RecordingProvider(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, spec: UpstreamEndpoint) -> None: + super().__init__(("127.0.0.1", 0), _RecordingHandler) + self.spec: Final = spec + self.interactions: queue.Queue[RecordedInteraction] = queue.Queue() + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + def take_interactions(self) -> tuple[RecordedInteraction, ...]: + try: + first: Final = self.interactions.get(timeout=5) + except queue.Empty as error: + raise RuntimeError("successful SDK call did not produce a recorded response") from error + remaining: Final = tuple(self.interactions.get_nowait() for _ in range(self.interactions.qsize())) + return (first, *remaining) + + +class _RecordingHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + self._forward() + + def do_GET(self) -> None: + self._forward() + + def do_PUT(self) -> None: + self._forward() + + def do_PATCH(self) -> None: + self._forward() + + def do_DELETE(self) -> None: + self._forward() + + def _forward(self) -> None: + provider: Final = self.server + assert isinstance(provider, _RecordingProvider) + length: Final = int(self.headers.get("content-length") or "0") + request_body: Final = self.rfile.read(length) if length else b"" + raw_headers: Final = tuple(self.headers.raw_items()) + excluded: Final = dropped_request_headers(raw_headers) + forwarded_headers: Final = tuple((name, value) for name, value in raw_headers if name.lower() not in excluded) + upstream_url: Final = f"{provider.spec.base_url.rstrip('/')}{self.path}" + + try: + with httpx.stream( + self.command, + upstream_url, + headers=forwarded_headers, + content=request_body, + timeout=120, + ) as upstream: + headers: Final = _end_to_end_headers(upstream.headers) + recorded_response: Final = self._record_upstream_response(upstream, headers) + except httpx.HTTPError as error: + self._send_response(502, (), str(error).encode("utf-8")) + return + + recorded_request: Final = remove_query_parameters( + Request( + self.command, + f"http://{_PARITY_PROVIDER_HOST}{self.path}", + request_body, + {name: value for name, value in forwarded_headers if name.lower() not in _SECRET_HEADERS}, + ), + ("api_key", "api-key", "key", "access_token", "subscription-key"), + ) + provider.interactions.put(RecordedInteraction(recorded_request, recorded_response)) + if isinstance(recorded_response, RecordedHttpResponse): + self._send_response( + recorded_response.status_code, recorded_response.headers, recorded_response.body_bytes() + ) + + def _record_upstream_response( + self, + upstream: httpx.Response, + headers: tuple[HttpHeader, ...], + ) -> RecordedResponse: + content_type: Final = cast(str, upstream.headers.get("content-type", "")) + if is_streaming_response(content_type): + return self._record_stream(upstream, headers) + response_body: Final = b"".join(upstream.iter_bytes()) + return RecordedHttpResponse.from_bytes( + status_code=upstream.status_code, + headers=headers, + body=response_body, + ) + + def _record_stream( + self, + upstream: httpx.Response, + headers: tuple[HttpHeader, ...], + ) -> RecordedHttpStreamResponse: + self.send_response_only(upstream.status_code) + provider: Final = self.server + assert isinstance(provider, _RecordingProvider) + for header in headers: + self.send_header(header.name, local_response_header(header.name, header.value, provider.url)) + self.send_header("transfer-encoding", "chunked") + self.end_headers() + chunks: Final = tuple(self._relay_chunks(upstream.iter_bytes())) + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + return RecordedHttpStreamResponse( + kind="http_stream", + status_code=upstream.status_code, + headers=headers, + chunks=chunks, + ) + + def _relay_chunks(self, chunks: Iterable[bytes]) -> Generator[RecordedStreamChunk, None, None]: + for chunk in chunks: + self.wfile.write(f"{len(chunk):X}\r\n".encode("ascii")) + self.wfile.write(chunk) + self.wfile.write(b"\r\n") + self.wfile.flush() + yield RecordedStreamChunk.from_bytes(chunk) + + def _send_response(self, status_code: int, headers: tuple[HttpHeader, ...], body: bytes) -> None: + self.send_response_only(status_code) + provider: Final = self.server + assert isinstance(provider, _RecordingProvider) + for header in headers: + self.send_header(header.name, local_response_header(header.name, header.value, provider.url)) + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + +@contextmanager +def _recording_provider(spec: UpstreamEndpoint) -> Generator[_RecordingProvider]: + server: Final = _RecordingProvider(spec) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def _invoke_and_take_interactions( + recorder: _RecordingProvider, + case_input: InputT, + sdk_call: Callable[[str, InputT], object], +) -> tuple[RecordedInteraction, ...]: + try: + sdk_call(recorder.url, case_input) + except Exception as invocation_error: + try: + return recorder.take_interactions() + except RuntimeError: + raise invocation_error + return recorder.take_interactions() + + +def record_upstream_interactions( + spec: UpstreamEndpoint, + case_input: InputT, + sdk_call: Callable[[str, InputT], object], +) -> tuple[RecordedInteraction, ...]: + with _recording_provider(spec) as recorder: + return _invoke_and_take_interactions(recorder, case_input, sdk_call) + + +def record_upstream_responses( + spec: UpstreamEndpoint, + case_input: InputT, + sdk_call: Callable[[str, InputT], object], +) -> tuple[RecordedResponse, ...]: + return tuple(item.response for item in record_upstream_interactions(spec, case_input, sdk_call)) diff --git a/tests/route_parity/fixtures/store.py b/tests/route_parity/fixtures/store.py new file mode 100644 index 00000000000..ca1ba58643d --- /dev/null +++ b/tests/route_parity/fixtures/store.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import hashlib +import json +import tempfile +from collections.abc import Mapping +from datetime import datetime, timezone +from pathlib import Path +from typing import Final, Literal, Protocol, TypeVar, cast + +from pydantic import AwareDatetime, BaseModel, ConfigDict, TypeAdapter, ValidationError + +from tests.route_parity.fixtures.cassette import deserialize_cassette, serialize_cassette +from tests.route_parity.fixtures.recording import RecordedInteraction + +FIXTURE_SCHEMA_VERSION: Final = 1 +JSON_OBJECT: Final = TypeAdapter(dict[str, object]) + + +class FixtureInput(Protocol): + def canonical_input(self) -> dict[str, object]: ... + + +CaseT = TypeVar("CaseT", bound=BaseModel) + + +class FixtureEnvelope(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: int + recorded_at: AwareDatetime + case: dict[str, object] + + +def canonical_json(value: Mapping[str, object]) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + +def fixture_cache_key(case_input: FixtureInput) -> dict[str, object]: + return case_input.canonical_input() + + +def fixture_path(directory: Path, case_input: FixtureInput) -> Path: + input_json: Final = canonical_json(fixture_cache_key(case_input)) + digest: Final = hashlib.sha256(input_json.encode("utf-8")).hexdigest() + return directory / f"{digest}.yaml" + + +def load_fixture(directory: Path, case_input: FixtureInput, case_type: type[CaseT]) -> CaseT | None: + path: Final = fixture_path(directory, case_input) + if path.is_file(): + return read_fixture(path, case_type) + legacy_path: Final = path.with_suffix(".json") + if not legacy_path.is_file(): + return None + return read_fixture(legacy_path, case_type) + + +def save_fixture( + directory: Path, + case_input: FixtureInput, + case: BaseModel, + interactions: tuple[RecordedInteraction, ...], + *, + recorded_at: datetime | None = None, + request_source: Literal["recorded", "python_replay"] = "recorded", +) -> Path: + directory.mkdir(parents=True, exist_ok=True) + path: Final = fixture_path(directory, case_input) + serialized: Final = serialize_cassette( + cast(dict[str, object], case.model_dump(mode="json", exclude_unset=True)), + interactions, + recorded_at or datetime.now(timezone.utc), + request_source, + ) + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=directory, delete=False) as temporary: + temporary_path: Final = Path(temporary.name) + try: + temporary.write(serialized) + temporary.close() + temporary_path.replace(path) + finally: + temporary_path.unlink(missing_ok=True) + return path + + +def read_fixture(path: Path, case_type: type[CaseT]) -> CaseT: + contents: Final = path.read_text(encoding="utf-8") + if path.suffix == ".json": + return _load_fixture(JSON_OBJECT.validate_json(contents), path, case_type) + try: + cassette: Final = deserialize_cassette(contents) + return case_type.model_validate(cassette.case_data()) + except ValueError as error: + raise ValueError(f"invalid parity cassette {path}") from error + + +def _load_fixture(raw_fixture: dict[str, object], path: Path, case_type: type[CaseT]) -> CaseT: + schema_version: Final = raw_fixture.get("schema_version") + if schema_version != FIXTURE_SCHEMA_VERSION: + raise ValueError( + f"fixture {path} has schema_version {schema_version!r}, expected {FIXTURE_SCHEMA_VERSION}; " + "delete it and regenerate the fixture bundle" + ) + try: + envelope: Final = FixtureEnvelope.model_validate(raw_fixture) + return case_type.model_validate(envelope.case) + except ValidationError as error: + raise ValueError(f"invalid parity fixture {path} ({len(error.errors())} validation errors)") from error + + +def recorded_fixtures(directory: Path, case_type: type[CaseT]) -> tuple[CaseT, ...]: + if not directory.is_dir(): + return () + paths: Final = tuple(sorted((*directory.rglob("*.yaml"), *directory.rglob("*.json")))) + return tuple(read_fixture(path, case_type) for path in paths) + + +def fixture_directory(configured: Path | None, env_value: str | None, default: Path) -> Path: + return (configured or Path(env_value or default)).expanduser() + + +def fixture_id(case_input: FixtureInput, prefix: str) -> str: + input_json: Final = canonical_json(case_input.canonical_input()) + digest: Final = hashlib.sha256(input_json.encode("utf-8")).hexdigest()[:8] + return f"{prefix}-{digest}" diff --git a/tests/route_parity/fixtures/test_cassette.py b/tests/route_parity/fixtures/test_cassette.py new file mode 100644 index 00000000000..d455a9a4d91 --- /dev/null +++ b/tests/route_parity/fixtures/test_cassette.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import Final + +import httpx +import pytest +from vcr import VCR +from vcr.request import Request + +from tests.route_parity.fixture_models import ParityCase, SdkInputBase +from tests.route_parity.fixtures.cassette import deserialize_cassette +from tests.route_parity.fixtures.recording import RecordedInteraction +from tests.route_parity.fixtures.store import load_fixture, save_fixture +from tests.route_parity.recorded_http import ( + HttpHeader, + RecordedHttpResponse, + RecordedHttpStreamResponse, + RecordedResponse, + RecordedStreamChunk, +) +from tests.route_parity.replay import replay_server + +_URI: Final = "http://parity-provider.invalid/operation?api-version=1" + + +class _Input(SdkInputBase): + model: str = "fixture-model" + + +@pytest.mark.parametrize("body", (b'{"text":"caf\xc3\xa9"}', b"\x00\xff\x80", b"")) +def test_cassette_replays_repeated_requests_with_vcr_and_preserves_bytes(tmp_path: Path, body: bytes) -> None: + sdk_input: Final = _Input() + responses: Final = tuple( + RecordedHttpResponse.from_bytes( + status, + (HttpHeader(name="content-type", value="application/octet-stream"),), + body, + ) + for status in (200, 429) + ) + case: Final = ParityCase[_Input](litellm_input=sdk_input, provider_responses=responses) + interactions: Final = tuple( + RecordedInteraction(Request("POST", _URI, b"\xffrequest", {}), response) for response in responses + ) + timestamp: Final = datetime(2020, 1, 1, tzinfo=timezone.utc) + path: Final = save_fixture(tmp_path, sdk_input, case, interactions, recorded_at=timestamp) + + assert load_fixture(tmp_path, sdk_input, ParityCase[_Input]) == case + assert deserialize_cassette(path.read_text()).recorded_at == timestamp + with VCR().use_cassette(str(path), record_mode="none", match_on=("method", "uri", "body")) as cassette: + for status in (200, 429): + replayed: Final = httpx.post(_URI, content=b"\xffrequest") + assert replayed.status_code == status + assert replayed.content == body + assert cassette.all_played + + +def test_stream_cassette_preserves_chunk_boundaries_through_local_replay(tmp_path: Path) -> None: + sdk_input: Final = _Input() + chunks: Final = (b"data: caf\xc3", b"\xa9\n\n", b"data: [DONE]\n\n") + response: Final = RecordedHttpStreamResponse( + kind="http_stream", + status_code=200, + headers=(HttpHeader(name="content-type", value="text/event-stream"),), + chunks=tuple(RecordedStreamChunk.from_bytes(chunk) for chunk in chunks), + ) + case: Final = ParityCase[_Input](litellm_input=sdk_input, provider_responses=(response,)) + path: Final = save_fixture( + tmp_path, sdk_input, case, (RecordedInteraction(Request("POST", _URI, b"{}", {}), response),) + ) + loaded: Final = load_fixture(tmp_path, sdk_input, ParityCase[_Input]) + assert loaded == case + with replay_server() as server: + server.enqueue_response(loaded.provider_responses[0]) + with httpx.stream("POST", f"{server.url}/operation", content=b"{}") as replayed: + assert tuple(replayed.iter_raw()) == chunks + server.take_requests(1) + path.write_text(path.read_text().replace("- 10\n", "- 999\n")) + with pytest.raises(ValueError, match="invalid parity cassette"): + load_fixture(tmp_path, sdk_input, ParityCase[_Input]) + + +def test_cassette_preserves_duplicate_response_headers(tmp_path: Path) -> None: + sdk_input: Final = _Input() + response: Final[RecordedResponse] = RecordedHttpResponse.from_bytes( + 200, + (HttpHeader(name="x-test", value="first"), HttpHeader(name="x-test", value="second")), + b"{}", + ) + case: Final = ParityCase[_Input](litellm_input=sdk_input, provider_responses=(response,)) + save_fixture(tmp_path, sdk_input, case, (RecordedInteraction(Request("POST", _URI, b"", {}), response),)) + + assert load_fixture(tmp_path, sdk_input, ParityCase[_Input]) == case diff --git a/tests/route_parity/fixtures/test_inputs.py b/tests/route_parity/fixtures/test_inputs.py new file mode 100644 index 00000000000..76081771b31 --- /dev/null +++ b/tests/route_parity/fixtures/test_inputs.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from typing import Final + +from hypothesis import strategies as st +from pydantic import BaseModel, ConfigDict + +from tests.route_parity.fixtures.inputs import generate_case_inputs + + +class _Input(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + identifier: str + + +def test_generate_case_inputs_is_deterministic() -> None: + strategy: Final = st.builds(_Input, identifier=st.integers().map(str)) + + assert generate_case_inputs(strategy, examples=4) == generate_case_inputs(strategy, examples=4) diff --git a/tests/route_parity/fixtures/test_media.py b/tests/route_parity/fixtures/test_media.py new file mode 100644 index 00000000000..5944074bf11 --- /dev/null +++ b/tests/route_parity/fixtures/test_media.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import base64 +from io import BytesIO +from typing import Final, cast + +from PIL import Image + +from tests.route_parity.fixtures.media import dummy_image_url, structured_image_bytes, structured_image_data_uri + + +def test_dummy_image_url_encodes_text_and_dimensions() -> None: + assert dummy_image_url("invoice 123", 24, width=320, height=80) == ( + "https://dummyjson.com/image/320x80/ffffff/000000?text=invoice%20123&fontSize=24" + ) + + +def test_structured_image_is_local_content_bearing_png() -> None: + png: Final = structured_image_bytes() + encoded: Final = structured_image_data_uri().partition(",")[2] + image: Final = Image.open(BytesIO(png)) + colors: Final = cast(list[tuple[int, tuple[int, int, int]]], image.getcolors(maxcolors=2)) + + assert png.startswith(b"\x89PNG\r\n\x1a\n") + assert base64.b64decode(encoded, validate=True) == png + assert image.size == (320, 80) + assert {color for _, color in colors} == {(0, 0, 0), (255, 255, 255)} diff --git a/tests/route_parity/fixtures/test_pipeline.py b/tests/route_parity/fixtures/test_pipeline.py new file mode 100644 index 00000000000..59de7ffbba7 --- /dev/null +++ b/tests/route_parity/fixtures/test_pipeline.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import logging +import threading +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final, Literal + +import httpx +import pytest +from hypothesis import strategies as st +from pydantic import BaseModel, ConfigDict + +from tests.route_parity.fixtures.pipeline import ( + RecordingInvocation, + RecordingTarget, + build_recording_jobs, + record_fixtures, +) +from tests.route_parity.fixtures.recording import UpstreamEndpoint +from tests.route_parity.fixtures.store import fixture_path +from tests.route_parity.recorded_http import RecordedResponse + + +class _FixtureInput(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + identifier: str + + def canonical_input(self) -> dict[str, object]: + return {"identifier": self.identifier} + + +class _ParityCase(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + litellm_input: _FixtureInput + provider_responses: tuple[RecordedResponse, ...] + + +class _Upstream(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), _UpstreamHandler) + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + +class _UpstreamHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + length: Final = int(self.headers.get("content-length") or "0") + self.rfile.read(length) + body: Final = b"{}" + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + +@contextmanager +def _upstream() -> Generator[_Upstream]: + server: Final = _Upstream() + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +@dataclass(frozen=True, slots=True) +class _OrderedInvocation: + order: Literal["slow", "fast"] + slow_started: threading.Event + fast_finished: threading.Event + + def execute(self, provider_url: str, case_input: _FixtureInput) -> None: + if self.order == "slow": + self.slow_started.set() + if not self.fast_finished.wait(timeout=2): + raise TimeoutError("fast recording did not finish") + else: + if not self.slow_started.wait(timeout=2): + raise TimeoutError("slow recording did not start") + response: Final = httpx.post(f"{provider_url}/record", json={"id": case_input.identifier}, timeout=5) + response.raise_for_status() + if self.order == "fast": + self.fast_finished.set() + + +@dataclass(frozen=True, slots=True) +class _Invocation: + def execute(self, provider_url: str, case_input: _FixtureInput) -> None: + response: Final = httpx.post(f"{provider_url}/record", json={"id": case_input.identifier}, timeout=5) + response.raise_for_status() + + +def _target( + name: str, + upstream_url: str, + case_input: _FixtureInput, + invocation: RecordingInvocation[_FixtureInput], +) -> RecordingTarget[_FixtureInput]: + return RecordingTarget( + name=name, + upstream=UpstreamEndpoint(base_url=upstream_url), + strategy=st.just(case_input), + invocation=invocation, + required_inputs=(case_input,), + ) + + +def test_build_jobs_keeps_required_inputs_before_generated_inputs_and_deduplicates(tmp_path: Path) -> None: + required: Final = _FixtureInput(identifier="required") + generated: Final = _FixtureInput(identifier="generated") + target: Final = RecordingTarget( + name="ordered", + upstream=UpstreamEndpoint(base_url="https://provider.invalid"), + strategy=st.just(generated), + invocation=_Invocation(), + required_inputs=(required, required), + ) + + jobs: Final = build_recording_jobs((target,), tmp_path, examples=1) + + assert tuple(job.case_input.identifier for job in jobs) == ("required", "generated") + + +def test_progress_follows_completion_order(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + slow_started: Final = threading.Event() + fast_finished: Final = threading.Event() + with _upstream() as upstream: + targets: Final = ( + _target( + "slow", + upstream.url, + _FixtureInput(identifier="slow"), + _OrderedInvocation("slow", slow_started, fast_finished), + ), + _target( + "fast", + upstream.url, + _FixtureInput(identifier="fast"), + _OrderedInvocation("fast", slow_started, fast_finished), + ), + ) + with caplog.at_level(logging.INFO, logger="tests.route_parity.fixtures.pipeline"): + summary: Final = record_fixtures(targets, tmp_path, 1, 2, _ParityCase) + + progress: Final = tuple(record.message for record in caplog.records if record.message.startswith("[")) + assert len(summary.recorded) == 2 + assert summary.exit_code == 0 + assert "recorded fast" in progress[0] + assert "recorded slow" in progress[1] + assert caplog.records[0].message == "Recording 2 fixtures across 2 targets with concurrency 2" + assert caplog.records[-1].message == "Finished 2 fixtures: 2 recorded, 0 cached, 0 failed" + + +def test_failure_does_not_stop_independent_recordings(tmp_path: Path) -> None: + stale_input: Final = _FixtureInput(identifier="stale") + stale_directory: Final = tmp_path / "stale" + stale_directory.mkdir() + fixture_path(stale_directory, stale_input).with_suffix(".json").write_text( + '{"schema_version": 0}\n', encoding="utf-8" + ) + with _upstream() as upstream: + targets: Final = ( + _target("stale", upstream.url, stale_input, _Invocation()), + _target("valid", upstream.url, _FixtureInput(identifier="valid"), _Invocation()), + ) + summary: Final = record_fixtures(targets, tmp_path, 1, 2, _ParityCase) + + assert len(summary.recorded) == 1 + assert summary.recorded[0].target_name == "valid" + assert len(summary.failed) == 1 + assert summary.failed[0].target_name == "stale" + assert summary.exit_code == 1 diff --git a/tests/route_parity/fixtures/test_recording.py b/tests/route_parity/fixtures/test_recording.py new file mode 100644 index 00000000000..f02f84d579b --- /dev/null +++ b/tests/route_parity/fixtures/test_recording.py @@ -0,0 +1,574 @@ +from __future__ import annotations + +import asyncio +import queue +import threading +from collections.abc import AsyncIterator, Callable, Generator, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final, Literal + +import httpx +import pytest +from hypothesis import strategies as st +from openai._streaming import SSEDecoder +from pydantic import BaseModel, ConfigDict + +from tests.route_parity.compare import assert_request_parity +from tests.route_parity.fixtures.pipeline import RecordingTarget, record_fixtures +from tests.route_parity.fixtures.recording import ( + UpstreamEndpoint, + record_upstream_interactions, + record_upstream_responses, +) +from tests.route_parity.fixtures.store import ( + FIXTURE_SCHEMA_VERSION, + fixture_path, + load_fixture, + recorded_fixtures, +) +from tests.route_parity.inprocess import InProcessExecution, run_in_process, run_in_process_async +from tests.route_parity.recorded_http import ( + HttpHeader, + RecordedHttpStreamResponse, + RecordedResponse, + RecordedStreamChunk, +) +from tests.route_parity.replay import ReplayServer, replay_server +from tests.route_parity.stream import ( + StreamCompleted, + StreamFailed, + StreamOutcome, + assert_stream_parity, + consume_async_stream, + consume_sync_stream, +) + +_SSE_CHUNKS: Final = ( + b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n', + b'data: {"choices":[{"delta":{"content":" world"}}]}\n\n', + b"data: [DONE]\n\n", +) + + +class _StreamEvent(BaseModel): + kind: Literal["delta", "done", "error"] + value: str + + +class _StreamApplicationError(Exception): + status_code: Final = 400 + code: Final = "invalid_input" + type: Final = "validation_error" + param: Final = "input" + model: Final = "fixture-model" + llm_provider: Final = "fixture-provider" + + +def _stream_event(data: str) -> _StreamEvent: + event: Final = _StreamEvent.model_validate_json(data) + if event.kind == "error": + raise _StreamApplicationError(event.value) + return event + + +def _event_chunks(failed: bool) -> tuple[bytes, ...]: + terminal: Final = ( + b'event: error\r\ndata: {"kind":"error","value":"invalid input"}\r\n\r\n' + if failed + else b'event: done\r\ndata: {"kind":"done","value":""}\r\n\r\n' + ) + return ( + b'event: delta\r\ndata: {"kind":"delta",\r\ndata: "value":"caf\xc3', + b'\xa9"}\r\n', + b'\r\nevent: delta\r\ndata: {"kind":"delta","value":"second"}\r\n\r\n' + terminal, + ) + + +def _sync_events(api_base: str, case_input: _FixtureInput) -> Iterator[_StreamEvent]: + with httpx.stream("POST", f"{api_base}/stream", json={"id": case_input.identifier}, timeout=5) as response: + response.raise_for_status() + for event in SSEDecoder().iter_bytes(response.iter_bytes()): + yield _stream_event(event.data) + + +async def _async_events(api_base: str, case_input: _FixtureInput) -> AsyncIterator[_StreamEvent]: + async with httpx.AsyncClient(timeout=5) as client: + async with client.stream("POST", f"{api_base}/stream", json={"id": case_input.identifier}) as response: + response.raise_for_status() + async for event in SSEDecoder().aiter_bytes(response.aiter_bytes()): + yield _stream_event(event.data) + + +async def _consume_async_events(api_base: str, case_input: _FixtureInput) -> StreamOutcome: + async def create() -> AsyncIterator[_StreamEvent]: + return _async_events(api_base, case_input) + + return await consume_async_stream(create) + + +async def _replay_events( + mode: Literal["sync", "async"], + provider: ReplayServer, + response: RecordedHttpStreamResponse, + case_input: _FixtureInput, +) -> InProcessExecution[StreamOutcome]: + if mode == "sync": + return run_in_process( + provider, (response,), lambda url: consume_sync_stream(lambda: _sync_events(url, case_input)) + ) + return await run_in_process_async(provider, (response,), lambda url: _consume_async_events(url, case_input)) + + +class _FixtureInput(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + identifier: str + + def canonical_input(self) -> dict[str, object]: + return {"identifier": self.identifier} + + +class _ParityCase(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + litellm_input: _FixtureInput + provider_responses: tuple[RecordedResponse, ...] + + +@dataclass(frozen=True, slots=True) +class _Invocation: + sdk_call: Callable[[str, _FixtureInput], object] + + def execute(self, provider_url: str, case_input: _FixtureInput) -> None: + self.sdk_call(provider_url, case_input) + + +class _ControlledUpstream(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, stream_chunks: tuple[bytes, ...]) -> None: + super().__init__(("127.0.0.1", 0), _ControlledUpstreamHandler) + self.stream_chunks: Final = stream_chunks + self.lock: Final = threading.Lock() + self.two_requests_started: Final = threading.Event() + self.active_requests: int = 0 + self.max_active_requests: int = 0 + self.request_count: int = 0 + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + def start_request(self) -> None: + with self.lock: + self.active_requests += 1 + self.request_count += 1 + self.max_active_requests = max(self.max_active_requests, self.active_requests) + if self.active_requests == 2: + self.two_requests_started.set() + self.two_requests_started.wait(timeout=2) + + def end_tracked_request(self) -> None: + with self.lock: + self.active_requests -= 1 + + +class _ControlledUpstreamHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + upstream: Final = self.server + assert isinstance(upstream, _ControlledUpstream) + length: Final = int(self.headers.get("content-length") or "0") + self.rfile.read(length) + if self.path == "/credentials?api_key=query-secret&api-version=1": + authorized: Final = self.headers.get("authorization") == "Bearer header-secret" + self._send_json(200 if authorized else 401, b"{}") + return + if self.path == "/upload": + self._send_json(200, b'{"file_id":"fixture://document.pdf"}') + return + if self.path == "/parse": + self._send_json(200, b'{"result":{"chunks":[]}}') + return + if self.path == "/analyze": + self.send_response(202) + self.send_header("operation-location", f"{upstream.url}/results/1") + self.send_header("content-length", "0") + self.end_headers() + return + if self.path in {"/v1/chat/completions", "/stream"}: + with upstream.lock: + upstream.request_count += 1 + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("transfer-encoding", "chunked") + self.end_headers() + for chunk in upstream.stream_chunks: + self.wfile.write(f"{len(chunk):X}\r\n".encode("ascii")) + self.wfile.write(chunk) + self.wfile.write(b"\r\n") + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + return + if self.path == "/error": + self._send_json(429, b'{"error":{"message":"rate limited"}}') + return + upstream.start_request() + try: + body: Final = b"{}" + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("set-cookie", "session=must-not-be-recorded") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + finally: + upstream.end_tracked_request() + + def do_GET(self) -> None: + if self.path == "/results/1": + self._send_json(200, b'{"status":"succeeded","analyzeResult":{"pages":[]}}') + return + self.send_error(404) + + def do_PUT(self) -> None: + self.do_POST() + + def do_PATCH(self) -> None: + self.do_POST() + + def do_DELETE(self) -> None: + self.do_POST() + + def _send_json(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + +@contextmanager +def _controlled_upstream(stream_chunks: tuple[bytes, ...] = _SSE_CHUNKS) -> Generator[_ControlledUpstream]: + server: Final = _ControlledUpstream(stream_chunks) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def _case(identifier: str) -> _FixtureInput: + return _FixtureInput(identifier=identifier) + + +def _sdk_call(api_base: str, case_input: _FixtureInput) -> object: + return httpx.post(f"{api_base}/v1/operation", content=b"{}", timeout=5) + + +def _stream_sdk_call(api_base: str, case_input: _FixtureInput) -> object: + return httpx.post(f"{api_base}/v1/chat/completions", content=b"{}", timeout=5) + + +def _error_sdk_call(api_base: str, case_input: _FixtureInput) -> object: + response: Final = httpx.post(f"{api_base}/error", content=b"{}", timeout=5) + response.raise_for_status() + return response + + +def _method_sdk_call(method: str) -> Callable[[str, _FixtureInput], object]: + def call(api_base: str, case_input: _FixtureInput) -> object: + return httpx.request(method, f"{api_base}/method", json={"id": case_input.identifier}, timeout=5) + + return call + + +def _multi_sdk_call(api_base: str, case_input: _FixtureInput) -> object: + upload: Final = httpx.post(f"{api_base}/upload", json={"document": case_input.identifier}, timeout=5) + upload.raise_for_status() + parsed: Final = httpx.post(f"{api_base}/parse", json={"input": upload.json()["file_id"]}, timeout=5) + parsed.raise_for_status() + return parsed + + +def _polling_sdk_call(api_base: str, case_input: _FixtureInput) -> object: + started: Final = httpx.post(f"{api_base}/analyze", json={"document": case_input.identifier}, timeout=5) + operation_location: Final = started.headers["operation-location"] + completed: Final = httpx.get(operation_location, timeout=5) + completed.raise_for_status() + return completed + + +def test_recording_deduplicates_per_target_and_caps_global_concurrency(tmp_path: Path) -> None: + shared_input: Final = _case("shared") + with _controlled_upstream() as upstream: + spec: Final = UpstreamEndpoint(base_url=upstream.url) + targets: Final = ( + RecordingTarget( + name="first", + upstream=spec, + strategy=st.just(shared_input), + invocation=_Invocation(_sdk_call), + required_inputs=(shared_input, shared_input), + ), + RecordingTarget( + name="second", + upstream=spec, + strategy=st.just(shared_input), + invocation=_Invocation(_sdk_call), + required_inputs=(shared_input,), + ), + ) + summary: Final = record_fixtures(targets, tmp_path, examples=1, concurrency=2, case_type=_ParityCase) + + assert len(summary.recorded) == 2 + assert {result.target_name for result in summary.recorded} == {"first", "second"} + assert summary.cached == () + assert summary.failed == () + assert upstream.request_count == 2 + assert upstream.max_active_requests == 2 + assert len(recorded_fixtures(tmp_path, _ParityCase)) == 2 + for path in tmp_path.rglob("*.yaml"): + contents = path.read_text(encoding="utf-8") + assert f"schema_version: {FIXTURE_SCHEMA_VERSION}" in contents + assert "recorded_at:" in contents + + +def test_pipeline_rejects_stale_fixture_before_provider_call(tmp_path: Path) -> None: + case_input: Final = _case("stale") + directory: Final = tmp_path / "stale-target" + directory.mkdir() + path: Final = fixture_path(directory, case_input).with_suffix(".json") + path.write_text('{"schema_version": 0}\n', encoding="utf-8") + target: Final = RecordingTarget( + name="stale-target", + upstream=UpstreamEndpoint(base_url="http://127.0.0.1:1"), + strategy=st.just(case_input), + invocation=_Invocation(_sdk_call), + ) + + summary: Final = record_fixtures( + (target,), + tmp_path, + examples=1, + concurrency=1, + case_type=_ParityCase, + ) + + assert summary.recorded == () + assert summary.cached == () + assert len(summary.failed) == 1 + assert str(summary.failed[0].error) == ( + f"fixture {path} has schema_version 0, expected {FIXTURE_SCHEMA_VERSION}; " + "delete it and regenerate the fixture bundle" + ) + + +def test_cached_fixture_is_reported_without_provider_call(tmp_path: Path) -> None: + case_input: Final = _case("cached") + with _controlled_upstream() as upstream: + target: Final = RecordingTarget( + name="cached-target", + upstream=UpstreamEndpoint(base_url=upstream.url), + strategy=st.just(case_input), + invocation=_Invocation(_sdk_call), + ) + first: Final = record_fixtures((target,), tmp_path, 1, 1, _ParityCase) + second: Final = record_fixtures((target,), tmp_path, 1, 1, _ParityCase) + + assert len(first.recorded) == 1 + assert len(second.cached) == 1 + assert upstream.request_count == 1 + + +def test_streaming_response_records_and_replays_chunks() -> None: + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case("stream"), + _stream_sdk_call, + ) + + response: Final = responses[0] + assert isinstance(response, RecordedHttpStreamResponse) + assert tuple(chunk.data_bytes() for chunk in response.chunks) == _SSE_CHUNKS + assert isinstance(response.model_dump(mode="json")["chunks"], list) + + with replay_server() as provider: + provider.enqueue_response(response) + with httpx.stream("POST", f"{provider.url}/v1/chat/completions", json={}) as replayed: + replayed_chunks: Final = tuple(replayed.iter_raw()) + provider.take_requests(1) + + assert replayed_chunks == _SSE_CHUNKS + + +def test_non_successful_provider_response_is_recorded() -> None: + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case("provider-error"), + _error_sdk_call, + ) + + response: Final = responses[0] + assert response.status_code == 429 + + +def test_sensitive_response_headers_are_not_recorded() -> None: + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case("headers"), + _sdk_call, + ) + + assert all(header.name.lower() != "set-cookie" for header in responses[0].headers) + + +def test_recorded_requests_strip_credentials_without_changing_the_live_request() -> None: + def sdk_call(api_base: str, case_input: _FixtureInput) -> object: + return httpx.post( + f"{api_base}/credentials?api_key=query-secret&api-version=1", + headers={ + "Authorization": "Bearer header-secret", + "Ocp-Apim-Subscription-Key": "azure-secret", + "Cookie": "session=cookie-secret", + "X-Test": case_input.identifier, + }, + content=b"\xffdocument", + ) + + with _controlled_upstream() as upstream: + interactions: Final = record_upstream_interactions( + UpstreamEndpoint(upstream.url), _case("credentials"), sdk_call + ) + + interaction: Final = interactions[0] + assert interaction.response.status_code == 200 + assert interaction.request.uri == "http://parity-provider.invalid/credentials?api-version=1" + assert interaction.request.body == b"\xffdocument" + assert interaction.request.headers["x-test"] == "credentials" + assert all( + header not in interaction.request.headers for header in ("authorization", "ocp-apim-subscription-key", "cookie") + ) + + +@pytest.mark.parametrize("method", ("PUT", "PATCH", "DELETE")) +def test_recording_and_replay_support_mutating_http_methods(method: str) -> None: + sdk_call: Final = _method_sdk_call(method) + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case(method), + sdk_call, + ) + with replay_server() as provider: + provider.enqueue_response(responses[0]) + sdk_call(provider.url, _case(method)) + requests: Final = provider.take_requests(1) + + assert requests[0].method == method + + +def test_stream_response_model_rejects_buffered_body() -> None: + with pytest.raises(ValueError, match="Extra inputs are not permitted"): + RecordedHttpStreamResponse.model_validate( + { + "kind": "http_stream", + "status_code": 200, + "headers": [HttpHeader(name="content-type", value="text/event-stream")], + "chunks": [RecordedStreamChunk.from_bytes(b"data: [DONE]\n\n")], + "body_b64": "", + } + ) + + +@pytest.mark.parametrize("sdk_call", (_multi_sdk_call, _polling_sdk_call)) +def test_multiple_provider_calls_record_and_replay_in_order( + sdk_call: Callable[[str, _FixtureInput], object], +) -> None: + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case(sdk_call.__name__), + sdk_call, + ) + + assert len(responses) == 2 + with replay_server() as provider: + for response in responses: + provider.enqueue_response(response) + sdk_call(provider.url, _case(sdk_call.__name__)) + requests: Final = provider.take_requests(2) + + assert len(requests) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize("failed", (False, True), ids=("completed", "application-error")) +async def test_typed_stream_recording_cassette_replay_parity( + tmp_path: Path, mode: Literal["sync", "async"], failed: bool +) -> None: + case_input: Final = _case("typed-stream") + outcomes: Final[queue.SimpleQueue[StreamOutcome]] = queue.SimpleQueue() + + def record(api_base: str, sdk_input: _FixtureInput) -> None: + outcome: Final = ( + consume_sync_stream(lambda: _sync_events(api_base, sdk_input)) + if mode == "sync" + else asyncio.run(_consume_async_events(api_base, sdk_input)) + ) + outcomes.put(outcome) + + with _controlled_upstream(_event_chunks(failed)) as upstream: + target: Final = RecordingTarget( + name="stream", + upstream=UpstreamEndpoint(upstream.url), + strategy=st.just(case_input), + invocation=_Invocation(record), + ) + summary: Final = record_fixtures((target,), tmp_path, 1, 1, _ParityCase) + + assert summary.failed == () + assert len(summary.recorded) == 1 + recorded: Final = outcomes.get_nowait() + loaded: Final = load_fixture(tmp_path / "stream", case_input, _ParityCase) + assert loaded is not None + response: Final = loaded.provider_responses[0] + assert isinstance(response, RecordedHttpStreamResponse) + assert response.status_code == 200 + wire_bytes: Final = b"".join(chunk.data_bytes() for chunk in response.chunks) + assert wire_bytes == b"".join(_event_chunks(failed)) + coalesced: Final = response.model_copy(update={"chunks": (RecordedStreamChunk.from_bytes(wire_bytes),)}) + + with replay_server() as provider: + first: Final = await _replay_events(mode, provider, response, case_input) + second: Final = await _replay_events(mode, provider, coalesced, case_input) + assert_request_parity(first.requests, second.requests) + assert len(first.requests) == 1 + assert first.requests[0].body == {"id": case_input.identifier} + assert_stream_parity(recorded, first.response) + assert_stream_parity(first.response, second.response) + expected: Final = (_StreamEvent(kind="delta", value="café"), _StreamEvent(kind="delta", value="second")) + assert first.response.chunks == (expected if failed else (*expected, _StreamEvent(kind="done", value=""))) + if failed: + assert isinstance(first.response.terminal, StreamFailed) + assert first.response.terminal.phase == "iteration" + assert first.response.terminal.exception_type is _StreamApplicationError + assert first.response.terminal.error.code == "invalid_input" + assert first.response.terminal.error.message == "invalid input" + else: + assert first.response.terminal == StreamCompleted() diff --git a/tests/route_parity/inprocess.py b/tests/route_parity/inprocess.py new file mode 100644 index 00000000000..6a11fd6a88d --- /dev/null +++ b/tests/route_parity/inprocess.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Final, Generic, TypeVar + +from tests.route_parity.models import CapturedRequest +from tests.route_parity.recorded_http import RecordedResponse +from tests.route_parity.replay import ReplayServer + +ResponseT = TypeVar("ResponseT") + + +@dataclass(frozen=True, slots=True) +class InProcessExecution(Generic[ResponseT]): + requests: tuple[CapturedRequest, ...] + response: ResponseT + + +def run_in_process( + provider: ReplayServer, + recorded_responses: tuple[RecordedResponse, ...], + call: Callable[[str], ResponseT], +) -> InProcessExecution[ResponseT]: + for recorded_response in recorded_responses: + provider.enqueue_response(recorded_response) + try: + response: Final = call(provider.url) + return InProcessExecution(requests=provider.take_requests(len(recorded_responses)), response=response) + except Exception: + provider.reset() + raise + + +async def run_in_process_async( + provider: ReplayServer, + recorded_responses: tuple[RecordedResponse, ...], + call: Callable[[str], Awaitable[ResponseT]], +) -> InProcessExecution[ResponseT]: + for recorded_response in recorded_responses: + provider.enqueue_response(recorded_response) + try: + response: Final = await call(provider.url) + return InProcessExecution(requests=provider.take_requests(len(recorded_responses)), response=response) + except Exception: + provider.reset() + raise diff --git a/tests/route_parity/models.py b/tests/route_parity/models.py new file mode 100644 index 00000000000..898b58d23ee --- /dev/null +++ b/tests/route_parity/models.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import base64 +from typing import Annotated, Final, Literal, cast + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter + + +class CapturedRequest(BaseModel): + model_config = ConfigDict(frozen=True) + + method: str + path: str + headers: tuple[tuple[str, str], ...] + body: JsonValue + user_agent: str | None + + +class SDKSuccess(BaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["ok"] = "ok" + response: JsonValue + + +class SDKError(BaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["error"] = "error" + exception_type: str + message: str + status_code: int | None + code: str | None + error_type: str | None + param: str | None + model: str | None + llm_provider: str | None + + +class SDKJsonChunk(BaseModel): + model_config = ConfigDict(frozen=True) + + kind: Literal["json"] = "json" + value: JsonValue + + +class SDKBytesChunk(BaseModel): + model_config = ConfigDict(frozen=True) + + kind: Literal["bytes"] = "bytes" + data_b64: str + + def data_bytes(self) -> bytes: + return base64.b64decode(self.data_b64, validate=True) + + +SDKChunk = Annotated[SDKJsonChunk | SDKBytesChunk, Field(discriminator="kind")] + + +class SDKStreamCompleted(BaseModel): + model_config = ConfigDict(frozen=True) + + kind: Literal["completed"] = "completed" + + +class SDKStreamFailed(BaseModel): + model_config = ConfigDict(frozen=True) + + kind: Literal["failed"] = "failed" + error: SDKError + + +SDKStreamTerminal = Annotated[SDKStreamCompleted | SDKStreamFailed, Field(discriminator="kind")] + + +class SDKStreamReport(BaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["stream"] = "stream" + chunks: tuple[SDKChunk, ...] + terminal: SDKStreamTerminal + + +SDKReport = Annotated[SDKSuccess | SDKError | SDKStreamReport, Field(discriminator="status")] +JSON_VALUE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +def sdk_chunk(value: object) -> SDKChunk: + if isinstance(value, bytes): + return SDKBytesChunk(data_b64=base64.b64encode(value).decode("ascii")) + if isinstance(value, BaseModel): + return SDKJsonChunk(value=JSON_VALUE_ADAPTER.validate_python(value.model_dump(mode="json"))) + return SDKJsonChunk(value=JSON_VALUE_ADAPTER.validate_python(value)) + + +def _string_attribute(error: Exception, name: str) -> str | None: + value: Final = cast(object | None, getattr(error, name, None)) + return None if value is None else str(value) + + +def sdk_error_report(error: Exception) -> SDKError: + message, _, _ = str(error).partition("\nTraceback (most recent call last):") + raw_status_code: Final = cast(object | None, getattr(error, "status_code", None)) + status_code: Final = raw_status_code if isinstance(raw_status_code, int) else None + return SDKError( + exception_type=f"{type(error).__module__}.{type(error).__qualname__}", + message=message.rstrip(), + status_code=status_code, + code=_string_attribute(error, "code"), + error_type=_string_attribute(error, "type"), + param=_string_attribute(error, "param"), + model=_string_attribute(error, "model"), + llm_provider=_string_attribute(error, "llm_provider"), + ) + + +class Execution(BaseModel): + model_config = ConfigDict(frozen=True) + + requests: tuple[CapturedRequest, ...] + report: SDKReport + + +class SDKCommand(BaseModel): + model_config = ConfigDict(frozen=True) + + case_file: str + route: str + + +class WorkerSuccess(BaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["ok"] = "ok" + report: SDKReport + + +class WorkerFailure(BaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["error"] = "error" + error: str + + +WorkerResult = Annotated[WorkerSuccess | WorkerFailure, Field(discriminator="status")] diff --git a/tests/route_parity/recorded_http.py b/tests/route_parity/recorded_http.py new file mode 100644 index 00000000000..940c5211753 --- /dev/null +++ b/tests/route_parity/recorded_http.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import base64 +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class _RecordedHttpModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class HttpHeader(_RecordedHttpModel): + name: str + value: str + + +class RecordedHttpResponse(_RecordedHttpModel): + kind: Literal["http"] + status_code: int + headers: tuple[HttpHeader, ...] + body_b64: str + + @classmethod + def from_bytes( + cls, + status_code: int, + headers: tuple[HttpHeader, ...], + body: bytes, + ) -> RecordedHttpResponse: + return cls( + kind="http", + status_code=status_code, + headers=headers, + body_b64=base64.b64encode(body).decode("ascii"), + ) + + def body_bytes(self) -> bytes: + return base64.b64decode(self.body_b64, validate=True) + + +class RecordedStreamChunk(_RecordedHttpModel): + data_b64: str + + @classmethod + def from_bytes(cls, data: bytes) -> RecordedStreamChunk: + return cls(data_b64=base64.b64encode(data).decode("ascii")) + + def data_bytes(self) -> bytes: + return base64.b64decode(self.data_b64, validate=True) + + +class RecordedHttpStreamResponse(_RecordedHttpModel): + kind: Literal["http_stream"] + status_code: int + headers: tuple[HttpHeader, ...] + chunks: tuple[RecordedStreamChunk, ...] + + +RecordedResponse = Annotated[ + RecordedHttpResponse | RecordedHttpStreamResponse, + Field(discriminator="kind"), +] diff --git a/tests/route_parity/replay.py b/tests/route_parity/replay.py new file mode 100644 index 00000000000..91a5e110393 --- /dev/null +++ b/tests/route_parity/replay.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import base64 +import queue +import threading +from collections.abc import Generator +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final + +from pydantic import JsonValue, TypeAdapter + +from tests.route_parity.fixtures.recording import local_response_header +from tests.route_parity.models import CapturedRequest +from tests.route_parity.recorded_http import RecordedHttpResponse, RecordedHttpStreamResponse, RecordedResponse + +JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +EXCLUDED_REQUEST_HEADERS: Final = frozenset( + { + "host", + "content-length", + "connection", + "accept-encoding", + "user-agent", + "x-litellm-parity-route", + } +) +EXCLUDED_RESPONSE_HEADERS: Final = frozenset({"content-length", "transfer-encoding", "connection"}) + + +class ReplayServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), _ReplayHandler) + self.responses: queue.Queue[RecordedResponse] = queue.Queue() + self.requests: queue.Queue[CapturedRequest] = queue.Queue() + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + def enqueue_response(self, response: RecordedResponse) -> None: + self.responses.put(response) + + def take_requests(self, expected_count: int) -> tuple[CapturedRequest, ...]: + request_count: Final = self.requests.qsize() + if request_count != expected_count: + raise AssertionError(f"expected exactly {expected_count} provider requests, received {request_count}") + return tuple(self.requests.get_nowait() for _ in range(request_count)) + + def reset(self) -> None: + while not self.responses.empty(): + self.responses.get_nowait() + while not self.requests.empty(): + self.requests.get_nowait() + + +class _ReplayHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + self._replay() + + def do_GET(self) -> None: + self._replay() + + def do_PUT(self) -> None: + self._replay() + + def do_PATCH(self) -> None: + self._replay() + + def do_DELETE(self) -> None: + self._replay() + + def _replay(self) -> None: + provider: Final = self.server + assert isinstance(provider, ReplayServer) + length: Final = int(self.headers.get("content-length") or "0") + raw_body: Final = self.rfile.read(length) if length else b"" + content_type: Final = self.headers.get("content-type", "") + body: Final = ( + JSON_VALUE.validate_json(raw_body) + if raw_body and content_type.lower().startswith("application/json") + else base64.b64encode(raw_body).decode("ascii") + if raw_body + else None + ) + headers: Final = tuple( + sorted( + (name.lower(), value) + for name, value in self.headers.raw_items() + if name.lower() not in EXCLUDED_REQUEST_HEADERS + ) + ) + provider.requests.put( + CapturedRequest( + method=self.command, + path=self.path, + headers=headers, + body=body, + user_agent=self.headers.get("user-agent"), + ) + ) + try: + response: Final = provider.responses.get(timeout=5) + except queue.Empty: + self.send_error(500, "no replay response queued") + return + self.send_response_only(response.status_code) + for header in response.headers: + if header.name.lower() not in EXCLUDED_RESPONSE_HEADERS: + self.send_header(header.name, local_response_header(header.name, header.value, provider.url)) + if isinstance(response, RecordedHttpResponse): + response_body: Final = response.body_bytes() + self.send_header("content-length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + return + assert isinstance(response, RecordedHttpStreamResponse) + self.send_header("transfer-encoding", "chunked") + self.end_headers() + for chunk in response.chunks: + data = chunk.data_bytes() + self.wfile.write(f"{len(data):X}\r\n".encode("ascii")) + self.wfile.write(data) + self.wfile.write(b"\r\n") + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + + def log_message(self, format: str, *args: object) -> None: + return + + +@contextmanager +def replay_server() -> Generator[ReplayServer]: + server: Final = ReplayServer() + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/tests/route_parity/runner.py b/tests/route_parity/runner.py new file mode 100644 index 00000000000..1b8d6242201 --- /dev/null +++ b/tests/route_parity/runner.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import asyncio +import os +import subprocess +import sys +from collections import deque +from collections.abc import Callable, Generator +from concurrent.futures import ThreadPoolExecutor, TimeoutError +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final, TextIO, cast + +from pydantic import TypeAdapter, ValidationError + +from tests.route_parity.models import ( + Execution, + SDKCommand, + WorkerFailure, + WorkerResult, + WorkerSuccess, +) +from tests.route_parity.recorded_http import RecordedResponse +from tests.route_parity.replay import ReplayServer, replay_server + +WORKER_RESULT_PREFIX: Final = "LITELLM_PARITY_RESULT " +WORKER_RESULT_ADAPTER: Final[TypeAdapter[WorkerResult]] = TypeAdapter(WorkerResult) + + +@dataclass(frozen=True, slots=True) +class SubprocessRunner: + entrypoint: Path + baseline_user_agent: str + route_label: str + + def command(self, provider_url: str) -> tuple[str, ...]: + return ( + sys.executable, + str(self.entrypoint.resolve()), + "--parity-worker", + provider_url, + ) + + +@dataclass(frozen=True, slots=True) +class ExecutionVariant: + name: str + environment: tuple[tuple[str, str], ...] + + +class SubprocessWorker: + def __init__(self, runner: SubprocessRunner, provider: ReplayServer, variant: ExecutionVariant) -> None: + project_root: Final = str(runner.entrypoint.resolve().parents[3]) + existing_pythonpath: Final = os.environ.get("PYTHONPATH") + env: Final = { + **os.environ, + **dict(variant.environment), + "LITELLM_USER_AGENT": runner.baseline_user_agent, + "PYTHONPATH": os.pathsep.join(path for path in (project_root, existing_pythonpath) if path), + } + self.mode: Final = variant.name + self.route_label: Final = runner.route_label + self.provider: Final = provider + self.process: Final = subprocess.Popen( + runner.command(provider.url), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=env, + ) + self.output_reader: Final = ThreadPoolExecutor(max_workers=1) + self.recent_output: Final[deque[str]] = deque(maxlen=100) + + def execute( + self, + case_file: Path, + route: str, + responses: tuple[RecordedResponse, ...], + ) -> Execution: + stdin: Final = self.process.stdin + if stdin is None or self.process.poll() is not None: + raise AssertionError(f"{self.mode} {self.route_label} worker exited before processing {case_file}") + for response in responses: + self.provider.enqueue_response(response) + command: Final = SDKCommand(case_file=str(case_file), route=route) + try: + stdin.write(f"{command.model_dump_json()}\n") + stdin.flush() + result: Final = self.output_reader.submit(self._read_result).result(timeout=60) + except TimeoutError as error: + self.provider.reset() + self.close() + raise AssertionError( + f"{self.mode} {self.route_label} worker timed out after 60s while processing {case_file}" + ) from error + except AssertionError: + self.provider.reset() + raise + except (BrokenPipeError, OSError) as error: + self.provider.reset() + raise AssertionError(self._failure_message(f"worker pipe failed while processing {case_file}")) from error + if isinstance(result, WorkerFailure): + self.provider.reset() + raise AssertionError( + f"{self.mode} {self.route_label} worker failed while processing {case_file}:\n{result.error}" + ) + assert isinstance(result, WorkerSuccess) + try: + return Execution(requests=self.provider.take_requests(len(responses)), report=result.report) + except AssertionError: + self.provider.reset() + raise + + def _read_result(self) -> WorkerResult: + process_stdout: Final = self.process.stdout + if process_stdout is None: + raise AssertionError(self._failure_message("worker stdout is unavailable")) + stdout: Final = cast(TextIO, process_stdout) + line: Final = stdout.readline() + if not line: + raise AssertionError(self._failure_message("worker exited without returning a result")) + stripped: Final = line.rstrip() + if not stripped.startswith(WORKER_RESULT_PREFIX): + self.recent_output.append(stripped) + return self._read_result() + payload: Final = stripped.removeprefix(WORKER_RESULT_PREFIX) + try: + return WORKER_RESULT_ADAPTER.validate_json(payload) + except ValidationError as error: + raise AssertionError(self._failure_message("worker returned an invalid result")) from error + + def _failure_message(self, message: str) -> str: + output: Final = "\n".join(self.recent_output) + prefix: Final = f"{self.mode} {self.route_label}" + return f"{prefix} {message}" if not output else f"{prefix} {message}\noutput:\n{output}" + + def close(self) -> None: + stdin: Final = self.process.stdin + if stdin is not None and not stdin.closed: + stdin.close() + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + self.process.terminate() + self.process.wait(timeout=10) + self.output_reader.shutdown(wait=True, cancel_futures=True) + + +@contextmanager +def execution_worker( + runner: SubprocessRunner, + variant: ExecutionVariant, +) -> Generator[SubprocessWorker]: + with replay_server() as provider: + worker: Final = SubprocessWorker(runner, provider, variant) + try: + yield worker + finally: + worker.close() + + +def run_execution( + worker: SubprocessWorker, + case_file: Path, + route: str, + responses: tuple[RecordedResponse, ...], +) -> Execution: + return worker.execute(case_file, route, responses) + + +@contextmanager +def execution_worker_pair( + runner: SubprocessRunner, + baseline: ExecutionVariant, + candidate: ExecutionVariant, +) -> Generator[tuple[SubprocessWorker, SubprocessWorker]]: + with execution_worker(runner, baseline) as baseline_worker: + with execution_worker(runner, candidate) as candidate_worker: + yield baseline_worker, candidate_worker + + +def parity_worker_main( + execute_command: Callable[[str, str, asyncio.AbstractEventLoop], WorkerResult], + mock_url: str, +) -> None: + event_loop: Final = asyncio.new_event_loop() + try: + for line in sys.stdin: + sys.stdout.write(f"{WORKER_RESULT_PREFIX}{execute_command(line, mock_url, event_loop).model_dump_json()}\n") + sys.stdout.flush() + finally: + event_loop.close() diff --git a/tests/route_parity/stream.py b/tests/route_parity/stream.py new file mode 100644 index 00000000000..e1bc7855f3d --- /dev/null +++ b/tests/route_parity/stream.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +from collections.abc import AsyncIterable, Awaitable, Callable, Iterable +from dataclasses import dataclass +from typing import Final, Literal, TypeAlias + +from tests.route_parity.compare import assert_value_parity +from tests.route_parity.models import ( + SDKError, + SDKReport, + SDKStreamCompleted, + SDKStreamFailed, + SDKStreamReport, + sdk_chunk, + sdk_error_report, +) + + +@dataclass(frozen=True, slots=True) +class StreamCompleted: + kind: Literal["completed"] = "completed" + + +@dataclass(frozen=True, slots=True) +class StreamFailed: + phase: Literal["creation", "iteration"] + exception_type: type[BaseException] + error: SDKError + kind: Literal["failed"] = "failed" + + +StreamTerminal: TypeAlias = StreamCompleted | StreamFailed + + +@dataclass(frozen=True, slots=True) +class StreamOutcome: + wrapper_type: type[object] | None + supports_sync_iteration: bool | None + supports_async_iteration: bool | None + chunks: tuple[object, ...] + chunk_types: tuple[type[object], ...] + terminal: StreamTerminal + + +ChunkNormalizer: TypeAlias = Callable[[object], object] + + +def drain_sync_stream(stream: Iterable[object]) -> None: + for _ in stream: + pass + + +async def drain_async_stream(stream: AsyncIterable[object]) -> None: + async for _ in stream: + pass + + +def capture_sync_stream(create: Callable[[], Iterable[object]]) -> SDKReport: + return _stream_report(consume_sync_stream(create)) + + +async def capture_async_stream(create: Callable[[], Awaitable[AsyncIterable[object]]]) -> SDKReport: + return _stream_report(await consume_async_stream(create)) + + +def _stream_report(outcome: StreamOutcome) -> SDKReport: + terminal: Final = outcome.terminal + if isinstance(terminal, StreamFailed) and terminal.phase == "creation": + return terminal.error + return SDKStreamReport( + chunks=tuple(sdk_chunk(chunk) for chunk in outcome.chunks), + terminal=SDKStreamFailed(error=terminal.error) if isinstance(terminal, StreamFailed) else SDKStreamCompleted(), + ) + + +def _failed(phase: Literal["creation", "iteration"], error: Exception) -> StreamFailed: + return StreamFailed( + phase=phase, + exception_type=type(error), + error=sdk_error_report(error), + ) + + +def consume_sync_stream(create: Callable[[], Iterable[object]]) -> StreamOutcome: + try: + stream: Final = create() + except Exception as error: + return StreamOutcome( + wrapper_type=None, + supports_sync_iteration=None, + supports_async_iteration=None, + chunks=(), + chunk_types=(), + terminal=_failed("creation", error), + ) + + chunks: list[object] = [] # mutable-ok: iterator consumption builds an ordered trace + try: + for chunk in stream: + chunks.append(chunk) # noqa: PERF402 # partial trace is required if iteration raises + except Exception as error: + recorded: Final = tuple(chunks) + return StreamOutcome( + wrapper_type=type(stream), + supports_sync_iteration=hasattr(stream, "__iter__"), + supports_async_iteration=hasattr(stream, "__aiter__"), + chunks=recorded, + chunk_types=tuple(type(chunk) for chunk in recorded), + terminal=_failed("iteration", error), + ) + completed_chunks: Final = tuple(chunks) + return StreamOutcome( + wrapper_type=type(stream), + supports_sync_iteration=hasattr(stream, "__iter__"), + supports_async_iteration=hasattr(stream, "__aiter__"), + chunks=completed_chunks, + chunk_types=tuple(type(chunk) for chunk in completed_chunks), + terminal=StreamCompleted(), + ) + + +async def consume_async_stream(create: Callable[[], Awaitable[AsyncIterable[object]]]) -> StreamOutcome: + try: + stream: Final = await create() + except Exception as error: + return StreamOutcome( + wrapper_type=None, + supports_sync_iteration=None, + supports_async_iteration=None, + chunks=(), + chunk_types=(), + terminal=_failed("creation", error), + ) + + chunks: list[object] = [] # mutable-ok: iterator consumption builds an ordered trace + try: + async for chunk in stream: + chunks.append(chunk) + except Exception as error: + recorded: Final = tuple(chunks) + return StreamOutcome( + wrapper_type=type(stream), + supports_sync_iteration=hasattr(stream, "__iter__"), + supports_async_iteration=hasattr(stream, "__aiter__"), + chunks=recorded, + chunk_types=tuple(type(chunk) for chunk in recorded), + terminal=_failed("iteration", error), + ) + completed_chunks: Final = tuple(chunks) + return StreamOutcome( + wrapper_type=type(stream), + supports_sync_iteration=hasattr(stream, "__iter__"), + supports_async_iteration=hasattr(stream, "__aiter__"), + chunks=completed_chunks, + chunk_types=tuple(type(chunk) for chunk in completed_chunks), + terminal=StreamCompleted(), + ) + + +def normalize_chunk(chunk: object) -> object: + return chunk + + +def assert_stream_parity( + baseline: StreamOutcome, + candidate: StreamOutcome, + *, + normalize: ChunkNormalizer = normalize_chunk, +) -> None: + assert baseline.wrapper_type is candidate.wrapper_type + assert baseline.supports_sync_iteration is candidate.supports_sync_iteration + assert baseline.supports_async_iteration is candidate.supports_async_iteration + assert baseline.chunk_types == candidate.chunk_types + assert len(baseline.chunks) == len(candidate.chunks) + for index, (baseline_chunk, candidate_chunk) in enumerate(zip(baseline.chunks, candidate.chunks, strict=True)): + assert_value_parity(normalize(baseline_chunk), normalize(candidate_chunk), path=f"$.chunks[{index}]") + assert baseline.terminal == candidate.terminal diff --git a/tests/route_parity/test_parity.py b/tests/route_parity/test_parity.py new file mode 100644 index 00000000000..55ebdc654be --- /dev/null +++ b/tests/route_parity/test_parity.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +from types import MappingProxyType +from typing import Final + +import pytest +from pydantic import BaseModel, ConfigDict, JsonValue, PrivateAttr + +from tests.route_parity.compare import assert_model_parity, assert_parity +from tests.route_parity.models import CapturedRequest, Execution, SDKError, SDKSuccess, sdk_error_report + +SENTINEL: Final = "python-parity-fallback" + + +class _ComparableResponse(BaseModel): + value: str + _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) + + def set_hidden_param(self, key: str, value: object) -> None: + self._hidden_params[key] = value + + +class _DifferentResponse(BaseModel): + value: str + + +class _FloatResponse(BaseModel): + values: list[float] + + +class _PublicValue(BaseModel): + model_config = ConfigDict(extra="allow") + + value: object + + +class _PublicError(ValueError): + status_code: Final = 400 + + +def _execution(*, body: JsonValue = None, markdown: str = "same", user_agent: str | None = None) -> Execution: + return Execution( + requests=( + CapturedRequest( + method="POST", + path="/v1/test-route?mode=test", + headers=(("authorization", "Bearer test-key"), ("content-type", "application/json")), + body={"model": "test-model"} if body is None else body, + user_agent=user_agent, + ), + ), + report=SDKSuccess(response={"items": [{"text": markdown}], "model": "test-model"}), + ) + + +def test_parity_rejects_request_difference() -> None: + python: Final = _execution(user_agent=SENTINEL) + rust: Final = _execution(body={"model": "different"}, user_agent="litellm-rust") + + with pytest.raises(AssertionError): + assert_parity(python, rust, SENTINEL) + + +def test_parity_rejects_response_difference() -> None: + python: Final = _execution(user_agent=SENTINEL) + rust: Final = _execution(markdown="different", user_agent="litellm-rust") + + with pytest.raises(AssertionError): + assert_parity(python, rust, SENTINEL) + + +def test_parity_rejects_error_difference() -> None: + python: Final = Execution( + requests=(), + report=SDKError( + exception_type="litellm.exceptions.BadRequestError", + message="bad request", + status_code=400, + code=None, + error_type=None, + param=None, + model="test-model", + llm_provider="test-provider", + ), + ) + rust: Final = python.model_copy(update={"report": python.report.model_copy(update={"status_code": 500})}) + + with pytest.raises(AssertionError): + assert_parity(python, rust, SENTINEL) + + +def test_sdk_error_report_removes_traceback_but_keeps_public_fields() -> None: + error: Final = _PublicError("invalid input\nTraceback (most recent call last):\n unstable") + + report: Final = sdk_error_report(error) + + assert report.exception_type.endswith("._PublicError") + assert report.message == "invalid input" + assert report.status_code == 400 + + +def test_parity_rejects_rust_fallback() -> None: + python: Final = _execution(user_agent=SENTINEL) + rust: Final = _execution(user_agent=SENTINEL) + + with pytest.raises(AssertionError, match="fell back"): + assert_parity(python, rust, SENTINEL) + + +def test_model_parity_compares_public_values_and_ignores_private_attrs() -> None: + python: Final = _ComparableResponse(value="same") + rust: Final = _ComparableResponse(value="same") + python.set_hidden_param("litellm_call_id", "python-id") + rust.set_hidden_param("litellm_call_id", "rust-id") + + assert_model_parity(python, rust) + + +def test_model_parity_rejects_public_value_difference() -> None: + python: Final = _ComparableResponse(value="python") + rust: Final = _ComparableResponse(value="rust") + + with pytest.raises(AssertionError): + assert_model_parity(python, rust) + + +def test_model_parity_rejects_type_difference() -> None: + with pytest.raises(AssertionError): + assert_model_parity(_ComparableResponse(value="same"), _DifferentResponse(value="same")) + + +def test_model_parity_rejects_wire_float_rounding_difference() -> None: + with pytest.raises(AssertionError, match=r"\$\.values\[0\]"): + assert_model_parity( + _FloatResponse(values=[0.22590550796036835]), + _FloatResponse(values=[0.22590550796036837]), + ) + + +def test_model_parity_rejects_meaningful_float_difference() -> None: + with pytest.raises(AssertionError, match=r"\$\.values\[0\]"): + assert_model_parity( + _FloatResponse(values=[0.22590550796036835]), + _FloatResponse(values=[0.2259]), + ) + + +@pytest.mark.parametrize( + ("baseline", "candidate"), + ( + (_ComparableResponse(value="same"), {"value": "same"}), + (_ComparableResponse(value="same"), _DifferentResponse(value="same")), + (True, 1), + (1, 1.0), + (["same"], ("same",)), + ({"value": "same"}, MappingProxyType({"value": "same"})), + ({True: "same"}, {1: "same"}), + ), + ids=("model-dict", "model-class", "bool-int", "int-float", "list-tuple", "mapping-class", "key-type"), +) +def test_model_parity_rejects_nested_type_changes(baseline: object, candidate: object) -> None: + with pytest.raises(AssertionError, match=r"\$\.value\[0\]"): + assert_model_parity(_PublicValue(value=[baseline]), _PublicValue(value=[candidate])) + + +def test_model_parity_ignores_nested_private_attributes() -> None: + baseline: Final = _ComparableResponse(value="same") + candidate: Final = _ComparableResponse(value="same") + baseline.set_hidden_param("request_id", "baseline") + candidate.set_hidden_param("request_id", "candidate") + + assert_model_parity(_PublicValue(value={"nested": [baseline]}), _PublicValue(value={"nested": [candidate]})) + + +@pytest.mark.parametrize("extras", ({"provider_value": "changed"}, {}, {"provider_value": {"value": "same"}})) +def test_model_parity_compares_public_extras(extras: dict[str, object]) -> None: + baseline: Final = _PublicValue.model_validate({"value": None, "provider_value": _ComparableResponse(value="same")}) + candidate: Final = _PublicValue.model_validate({"value": None, **extras}) + + with pytest.raises(AssertionError): + assert_model_parity(baseline, candidate) + + +def test_serialized_parity_rejects_boolean_integer_substitution() -> None: + with pytest.raises(AssertionError, match="type mismatch"): + assert_parity( + _execution(body={"enabled": True}, user_agent=SENTINEL), + _execution(body={"enabled": 1}, user_agent="candidate"), + SENTINEL, + ) diff --git a/tests/route_parity/test_stream.py b/tests/route_parity/test_stream.py new file mode 100644 index 00000000000..cf39d8f557c --- /dev/null +++ b/tests/route_parity/test_stream.py @@ -0,0 +1,344 @@ +from __future__ import annotations + +import queue +from collections.abc import AsyncIterator, Iterator +from typing import Final, Literal, NoReturn + +import pytest +from pydantic import BaseModel, PrivateAttr, ValidationError + +from tests.route_parity.models import ( + SDKBytesChunk, + SDKError, + SDKJsonChunk, + SDKReport, + SDKStreamCompleted, + SDKStreamFailed, + SDKStreamReport, + sdk_error_report, +) +from tests.route_parity.stream import ( + StreamCompleted, + StreamFailed, + StreamOutcome, + assert_stream_parity, + capture_async_stream, + capture_sync_stream, + consume_async_stream, + consume_sync_stream, + drain_async_stream, + drain_sync_stream, +) + + +class _Chunk(BaseModel): + value: str + _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) + + def set_hidden_param(self, key: str, value: object) -> None: + self._hidden_params[key] = value + + +class _NestedChunk(BaseModel): + value: object + + +class _SyncStream: + def __init__(self, chunks: tuple[object, ...], error: BaseException | None = None) -> None: + self.chunks: Final = chunks + self.error: Final = error + + def __iter__(self) -> Iterator[object]: + yield from self.chunks + if self.error is not None: + raise self.error + + +class _AsyncStream: + def __init__(self, chunks: tuple[object, ...], error: BaseException | None = None) -> None: + self.chunks: Final = chunks + self.error: Final = error + + async def __aiter__(self) -> AsyncIterator[object]: + for chunk in self.chunks: + yield chunk + if self.error is not None: + raise self.error + + +class _PublicStreamError(Exception): + def __init__( + self, + message: str = "invalid input", + *, + status_code: int = 400, + llm_provider: str = "test", + model: str = "test-model", + code: str = "invalid_input", + error_type: str = "validation_error", + param: str = "input", + ) -> None: + super().__init__(message) + self.status_code: Final = status_code + self.llm_provider: Final = llm_provider + self.model: Final = model + self.code: Final = code + self.type: Final = error_type + self.param: Final = param + + +def _creation_error() -> NoReturn: + raise _PublicStreamError(status_code=429, llm_provider="test", model="test-model") + + +async def _async_stream(chunks: tuple[object, ...], error: BaseException | None = None) -> _AsyncStream: + return _AsyncStream(chunks, error) + + +async def _consume( + mode: Literal["sync", "async"], chunks: tuple[object, ...], error: Exception | None = None +) -> StreamOutcome: + if mode == "sync": + return consume_sync_stream(lambda: _SyncStream(chunks, error)) + return await consume_async_stream(lambda: _async_stream(chunks, error)) + + +async def _capture( + mode: Literal["sync", "async"], chunks: tuple[object, ...], error: Exception | None = None +) -> SDKReport: + if mode == "sync": + return capture_sync_stream(lambda: _SyncStream(chunks, error)) + return await capture_async_stream(lambda: _async_stream(chunks, error)) + + +def test_sync_stream_parity_compares_chunks_and_ignores_private_attrs() -> None: + python_chunk: Final = _Chunk(value="same") + accelerated_chunk: Final = _Chunk(value="same") + python_chunk.set_hidden_param("request_id", "python") + accelerated_chunk.set_hidden_param("request_id", "accelerated") + python: Final = consume_sync_stream(lambda: _SyncStream((python_chunk,))) + accelerated: Final = consume_sync_stream(lambda: _SyncStream((accelerated_chunk,))) + + assert python.supports_sync_iteration is True + assert python.supports_async_iteration is False + assert_stream_parity(python, accelerated) + + +def test_stream_parity_rejects_extra_chunk() -> None: + python: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="one"),))) + accelerated: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="one"), _Chunk(value="two")))) + + with pytest.raises(AssertionError): + assert_stream_parity(python, accelerated) + + +def test_stream_parity_rejects_chunk_value_difference() -> None: + python: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="python"),))) + accelerated: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="accelerated"),))) + + with pytest.raises(AssertionError): + assert_stream_parity(python, accelerated) + + +def test_stream_outcome_distinguishes_creation_and_iteration_errors() -> None: + creation: Final = consume_sync_stream(_creation_error) + iteration: Final = consume_sync_stream( + lambda: _SyncStream( + (_Chunk(value="before-error"),), + _PublicStreamError(status_code=429, llm_provider="test", model="test-model"), + ) + ) + + assert isinstance(creation.terminal, StreamFailed) + assert creation.terminal.phase == "creation" + assert creation.chunks == () + assert isinstance(iteration.terminal, StreamFailed) + assert iteration.terminal.phase == "iteration" + assert len(iteration.chunks) == 1 + with pytest.raises(AssertionError): + assert_stream_parity(creation, iteration) + + +@pytest.mark.asyncio +async def test_async_stream_uses_same_trace_contract() -> None: + python_error: Final = _PublicStreamError( + "invalid input\nTraceback (most recent call last):\npython detail", status_code=500 + ) + accelerated_error: Final = _PublicStreamError( + "invalid input\nTraceback (most recent call last):\nrust detail", status_code=500 + ) + python: Final = await consume_async_stream(lambda: _async_stream((_Chunk(value="same"),), python_error)) + accelerated: Final = await consume_async_stream(lambda: _async_stream((_Chunk(value="same"),), accelerated_error)) + + assert python.supports_sync_iteration is False + assert python.supports_async_iteration is True + assert_stream_parity(python, accelerated) + + +def test_stream_parity_accepts_route_specific_chunk_normalizer() -> None: + python: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="python-generated-id"),))) + accelerated: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="rust-generated-id"),))) + + assert_stream_parity(python, accelerated, normalize=lambda chunk: type(chunk)) + + +def test_drain_sync_stream_exhausts_lazy_iterator() -> None: + consumed: Final[queue.SimpleQueue[str]] = queue.SimpleQueue() + + def chunks() -> Iterator[object]: + yield _Chunk(value="one") + consumed.put("complete") + + drain_sync_stream(chunks()) + + assert consumed.get_nowait() == "complete" + + +@pytest.mark.asyncio +async def test_drain_async_stream_exhausts_lazy_iterator() -> None: + consumed: Final[queue.SimpleQueue[str]] = queue.SimpleQueue() + + async def chunks() -> AsyncIterator[object]: + yield b"one" + consumed.put("complete") + + await drain_async_stream(chunks()) + + assert consumed.get_nowait() == "complete" + + +def test_capture_sync_stream_serializes_model_chunks_and_partial_failure() -> None: + report: Final = capture_sync_stream( + lambda: _SyncStream( + (_Chunk(value="before-error"),), + _PublicStreamError(status_code=429, llm_provider="test", model="test-model"), + ) + ) + + assert isinstance(report, SDKStreamReport) + assert len(report.chunks) == 1 + chunk: Final = report.chunks[0] + assert isinstance(chunk, SDKJsonChunk) + assert chunk.value == {"value": "before-error"} + assert isinstance(report.terminal, SDKStreamFailed) + assert report.terminal.error.status_code == 429 + + +@pytest.mark.asyncio +async def test_capture_async_stream_serializes_message_bytes_in_order() -> None: + report: Final = await capture_async_stream(lambda: _async_stream((b"first", b"second"))) + + assert isinstance(report, SDKStreamReport) + assert tuple(chunk.data_bytes() for chunk in report.chunks if isinstance(chunk, SDKBytesChunk)) == ( + b"first", + b"second", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize( + "candidate_error", + ( + ValueError("invalid input"), + _PublicStreamError("changed message"), + _PublicStreamError(status_code=429), + _PublicStreamError(code="changed_code"), + _PublicStreamError(error_type="changed_type"), + _PublicStreamError(param="changed_param"), + _PublicStreamError(model="changed_model"), + _PublicStreamError(llm_provider="changed_provider"), + ), + ids=("exception", "message", "status", "code", "type", "param", "model", "provider"), +) +async def test_stream_parity_rejects_public_error_changes( + mode: Literal["sync", "async"], candidate_error: Exception +) -> None: + chunks: Final = (_Chunk(value="partial"),) + baseline: Final = await _consume(mode, chunks, _PublicStreamError()) + candidate: Final = await _consume(mode, chunks, candidate_error) + + with pytest.raises(AssertionError): + assert_stream_parity(baseline, candidate) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize("candidate", (("one",), ("one", "two", "three"), ("two", "one"), ("one", "changed"))) +async def test_stream_parity_checks_event_sequence(mode: Literal["sync", "async"], candidate: tuple[str, ...]) -> None: + baseline: Final = await _consume(mode, (_Chunk(value="one"), _Chunk(value="two"))) + changed: Final = await _consume(mode, tuple(_Chunk(value=value) for value in candidate)) + + with pytest.raises(AssertionError): + assert_stream_parity(baseline, changed) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_stream_parity_preserves_nested_types_and_ignores_private_fields(mode: Literal["sync", "async"]) -> None: + first: Final = _Chunk(value="same") + second: Final = _Chunk(value="same") + first.set_hidden_param("request_id", "first") + second.set_hidden_param("request_id", "second") + baseline: Final = await _consume(mode, (_NestedChunk(value=[first]),)) + candidate: Final = await _consume(mode, (_NestedChunk(value=[second]),)) + + assert_stream_parity(baseline, candidate) + changed: Final = await _consume(mode, (_NestedChunk(value=[{"value": "same"}]),)) + with pytest.raises(AssertionError, match=r"\$\.chunks\[0\]\.value\[0\]"): + assert_stream_parity(baseline, changed) + + +def test_stream_parity_rejects_wrapper_and_chunk_type_changes() -> None: + baseline: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="same"),))) + different_wrapper: Final = consume_sync_stream(lambda: iter((_Chunk(value="same"),))) + different_chunk: Final = consume_sync_stream(lambda: _SyncStream(({"value": "same"},))) + + with pytest.raises(AssertionError): + assert_stream_parity(baseline, different_wrapper) + with pytest.raises(AssertionError): + assert_stream_parity(baseline, different_chunk) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize("error", (None, _PublicStreamError())) +async def test_capture_keeps_serialization_failures_out_of_sdk_errors( + mode: Literal["sync", "async"], error: Exception | None +) -> None: + with pytest.raises(ValidationError): + await _capture(mode, (_Chunk(value="valid"), object()), error) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_empty_stream_completes(mode: Literal["sync", "async"]) -> None: + outcome: Final = await _consume(mode, ()) + assert outcome.chunks == () + assert outcome.terminal == StreamCompleted() + assert await _capture(mode, ()) == SDKStreamReport(chunks=(), terminal=SDKStreamCompleted()) + + +@pytest.mark.asyncio +async def test_async_creation_error_matches_sync_capture() -> None: + async def create() -> _AsyncStream: + _creation_error() + + sync: Final = consume_sync_stream(_creation_error) + asynchronous: Final = await consume_async_stream(create) + assert_stream_parity(sync, asynchronous) + assert isinstance(sync.terminal, StreamFailed) + assert sync.terminal.phase == "creation" + assert isinstance(capture_sync_stream(_creation_error), SDKError) + assert capture_sync_stream(_creation_error) == await capture_async_stream(create) == sync.terminal.error + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_capture_preserves_partial_output_and_complete_error(mode: Literal["sync", "async"]) -> None: + error: Final = _PublicStreamError() + report: Final = await _capture(mode, (_Chunk(value="partial"),), error) + assert report == SDKStreamReport( + chunks=(SDKJsonChunk(value={"value": "partial"}),), + terminal=SDKStreamFailed(error=sdk_error_report(error)), + ) diff --git a/tests/test_litellm/ocr/conftest.py b/tests/test_litellm/ocr/conftest.py new file mode 100644 index 00000000000..a32963a5ab6 --- /dev/null +++ b/tests/test_litellm/ocr/conftest.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Final + +import pytest + +from tests.route_parity.fixtures.pytest_support import parametrize_recorded_fixtures +from tests.route_parity.fixtures.store import fixture_id +from tests.test_litellm.ocr.fixtures.config import DEFAULT_FIXTURE_DIRECTORY, FIXTURE_DIR_ENV +from tests.test_litellm.ocr.fixtures.models import OcrParityCase + + +def ocr_fixture_id(fixture: OcrParityCase) -> str: + case_input: Final = fixture.litellm_input + provider: Final = case_input.custom_llm_provider + prefix: Final = f"{provider}/{case_input.model}" if provider else case_input.model + return fixture_id(case_input, prefix) + + +def ocr_fixture_marks(fixture: OcrParityCase) -> tuple[pytest.MarkDecorator, ...]: + if fixture.litellm_input.contract not in {"reducto_v3", "reducto_legacy"}: + return () + return ( + pytest.mark.xfail( + reason="Reducto does not have a Rust OCR contract", + strict=False, + ), + ) + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + parametrize_recorded_fixtures( + metafunc, + fixture_name="ocr_fixture", + case_type=OcrParityCase, + env_var=FIXTURE_DIR_ENV, + default_directory=DEFAULT_FIXTURE_DIRECTORY, + regeneration_command=( + f"uv run python -m tests.test_litellm.ocr.fixtures.record --fixture-dir {DEFAULT_FIXTURE_DIRECTORY}" + ), + id_builder=ocr_fixture_id, + marks_builder=ocr_fixture_marks, + ) diff --git a/tests/test_litellm/ocr/fixtures/README.md b/tests/test_litellm/ocr/fixtures/README.md new file mode 100644 index 00000000000..a1b9c8c3a99 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/README.md @@ -0,0 +1,49 @@ +# OCR parity fixtures + +The recording command runs four stages: + +1. Generate deterministic SDK inputs for every configured OCR target +2. Build target-scoped, deduplicated recording jobs +3. Record upstream responses through one globally bounded worker pool +4. Persist each fixture and report whether it was recorded, cached, or failed + +Run it with: + +```shell +uv run python -m tests.test_litellm.ocr.fixtures.record --examples 4 --concurrency 4 +``` + +`--concurrency` caps provider calls across all targets. Independent jobs finish after a failure, then the command exits +nonzero if any job failed + +New recordings are VCR YAML cassettes. The committed corpus contains 31 migrated cassettes: 18 Mistral, 9 Reducto v3, +and 4 Reducto legacy. Their original response bytes, statuses, headers, and recording timestamps are preserved + +To migrate an existing JSON fixture directory locally: + +```shell +uv run python -m tests.test_litellm.ocr.fixtures.migrate --fixture-dir tests/test_litellm/ocr/fixtures/data +``` + +The migration replays each old response through the Python SDK to reconstruct missing requests, writes and validates +the YAML cassette, then removes its JSON predecessor. It calls only local recording/replay servers and needs no provider +credentials. Reconstructed requests are labeled `python_replay`; they are not historical wire captures. Filenames use +the current normalized SDK input hash, including the fixture contract + +OCR strategies generate public `litellm.ocr()` and `litellm.aocr()` inputs. Every case contains the normalized model, +document, optional provider override, and LiteLLM keyword arguments. The fixture-only `contract` literal selects the +input schema and is removed before calling the SDK. Strategies never build provider wire payloads + +Each contract has a required corpus containing a baseline and cases for its supported top-level OCR parameters. The +contracts are Mistral, Azure-hosted Mistral, Vertex-hosted Mistral, Azure Document Intelligence, Vertex DeepSeek, +Reducto v3, and Reducto legacy. Credentials and endpoints only control target discovery, so a machine records the +contracts it has configured and skips the rest + +Reducto fixtures record upload and parse responses. Their parity cases remain non-strict expected failures until the +Rust OCR bridge supports Reducto. Azure and Vertex generation paths are unit-tested without credentials in CI, so the +committed corpus does not need live recordings for every target + +Every recording target owns a small fixed provider-rejected corpus, independent of replay implementation support. +Those inputs are recorded separately from generated valid inputs. Local validation failures use no recorded response; +the parity suite checks those unsupported providers and models, malformed documents, invalid request formats, invalid +Azure Document Intelligence parameters, and invalid headers in sync and async SDK calls diff --git a/tests/test_litellm/ocr/fixtures/__init__.py b/tests/test_litellm/ocr/fixtures/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_litellm/ocr/fixtures/azure.py b/tests/test_litellm/ocr/fixtures/azure.py new file mode 100644 index 00000000000..50521a9c0e8 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/azure.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final, Literal, cast + +from hypothesis import strategies as st +from hypothesis.strategies import SearchStrategy +from pydantic import StrictInt, StrictStr, field_validator + +from tests.route_parity.fixtures.recording import UpstreamEndpoint +from tests.test_litellm.ocr.fixtures.base import OcrDocument, OcrSdkInputBase +from tests.test_litellm.ocr.fixtures.common import ( + OcrFixtureClient, + OcrRecordingTarget, + image_document, + invoke_with_api_key, + pdf_document, +) +from tests.test_litellm.ocr.fixtures.mistral import ( + MistralCompatibleOcrSdkInput, + mistral_input_values_strategy, +) + +AzureMistralModel = Literal["azure_ai/mistral-document-ai-2512",] +AzureMistralFixtureModel = AzureMistralModel | Literal["azure_ai/invalid-ocr-model-for-parity"] +AzureDocumentIntelligenceModel = Literal[ + "azure_ai/doc-intelligence/prebuilt-read", + "azure_ai/doc-intelligence/prebuilt-layout", + "azure_ai/doc-intelligence/prebuilt-document", +] +AzureDocumentIntelligenceFixtureModel = ( + AzureDocumentIntelligenceModel | Literal["azure_ai/doc-intelligence/invalid-ocr-model-for-parity"] +) + +AZURE_MISTRAL_MODELS: Final[tuple[AzureMistralModel, ...]] = ("azure_ai/mistral-document-ai-2512",) +AZURE_DOCUMENT_INTELLIGENCE_MODELS: Final[tuple[AzureDocumentIntelligenceModel, ...]] = ( + "azure_ai/doc-intelligence/prebuilt-read", + "azure_ai/doc-intelligence/prebuilt-layout", + "azure_ai/doc-intelligence/prebuilt-document", +) +# API v4 replaces prebuilt-document with prebuilt-layout plus keyValuePairs. Keep +# the broader fixture model above so existing recordings remain loadable. +AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS: Final[tuple[AzureDocumentIntelligenceModel, ...]] = ( + "azure_ai/doc-intelligence/prebuilt-read", + "azure_ai/doc-intelligence/prebuilt-layout", +) + + +class AzureMistralOcrSdkInput(MistralCompatibleOcrSdkInput): + contract: Literal["azure_mistral"] = "azure_mistral" + model: AzureMistralFixtureModel + custom_llm_provider: Literal["azure_ai"] | None = None + + @field_validator("model") + @classmethod + def validate_model_namespace(cls, model: str) -> str: + if not model.startswith("azure_ai/"): + raise ValueError("Azure Mistral models must use the azure_ai/ LiteLLM namespace") + return model + + +class AzureDocumentIntelligenceOcrSdkInput(OcrSdkInputBase): + contract: Literal["azure_document_intelligence"] = "azure_document_intelligence" + model: AzureDocumentIntelligenceFixtureModel + document: OcrDocument + custom_llm_provider: Literal["azure_ai"] | None = None + pages: str | list[StrictInt] | list[StrictStr] | None = None + features: str | list[str] | None = None + req_format: Literal["litellm"] = "litellm" + + +AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS: Final[tuple[AzureMistralOcrSdkInput, ...]] = ( + AzureMistralOcrSdkInput( + model="azure_ai/invalid-ocr-model-for-parity", + document=pdf_document(), + ), +) +AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS: Final[tuple[AzureDocumentIntelligenceOcrSdkInput, ...]] = ( + AzureDocumentIntelligenceOcrSdkInput( + model="azure_ai/doc-intelligence/invalid-ocr-model-for-parity", + document=pdf_document(), + ), +) + + +def _azure_mistral_input(values: dict[str, object], model: AzureMistralModel) -> AzureMistralOcrSdkInput: + return AzureMistralOcrSdkInput.model_validate({**values, "model": model}) + + +def azure_mistral_input_strategy(inline_image_data_uri: str) -> SearchStrategy[AzureMistralOcrSdkInput]: + # Foundry's active gateway schema rejects 2512-only controls and + # document_annotation_prompt, even though native Mistral accepts them. + return st.builds( + _azure_mistral_input, + values=mistral_input_values_strategy("2505", inline_image_data_uri, include_document_annotation_prompt=False), + model=st.sampled_from(AZURE_MISTRAL_MODELS), + ) + + +_AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL: Final[AzureDocumentIntelligenceModel] = ( + "azure_ai/doc-intelligence/prebuilt-layout" +) + + +def _document_intelligence_input( + model: AzureDocumentIntelligenceModel, + document: OcrDocument, + optional_params: Mapping[str, object] | None = None, +) -> AzureDocumentIntelligenceOcrSdkInput: + return AzureDocumentIntelligenceOcrSdkInput.model_validate( + {"model": model, "document": document, **(optional_params or {})} + ) + + +def azure_document_intelligence_input_strategy() -> SearchStrategy[AzureDocumentIntelligenceOcrSdkInput]: + document: Final = pdf_document() + pages: Final = st.one_of( + st.sampled_from(((0,), (2, 0, 0, 1))).map(list), + st.just(["1", "2-4"]), + st.just("1-4, 5"), + ).map(lambda value: {"pages": value}) + features: Final = st.one_of( + st.sampled_from( + ( + ("languages",), + ("ocrHighResolution",), + ("barcodes",), + ("formulas",), + ("styleFont",), + ("keyValuePairs",), + ) + ).map(list), + st.just("languages, styleFont"), + ).map(lambda value: {"features": value}) + combined_query: Final = st.just({"pages": (0, 1), "features": ("languages", "styleFont")}) + return st.one_of( + st.sampled_from(AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS).map( + lambda model: _document_intelligence_input(model, document) + ), + st.just( + _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, + image_document("invoice 123", 24), + ) + ), + pages.map( + lambda optional_params: _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params + ) + ), + features.map( + lambda optional_params: _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params + ) + ), + combined_query.map( + lambda optional_params: _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params + ) + ), + st.just( + _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, + document, + {"req_format": "litellm"}, + ) + ), + ) + + +def azure_mistral_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str +) -> tuple[OcrRecordingTarget, ...]: + api_key: Final = environ.get("AZURE_AI_API_KEY") + base_url: Final = environ.get("AZURE_AI_API_BASE") + if not api_key or not base_url: + return () + return ( + OcrRecordingTarget( + name="azure-mistral", + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), + strategy=cast( + SearchStrategy[OcrSdkInputBase], + azure_mistral_input_strategy(inline_image_data_uri), + ), + invocation=invoke_with_api_key(client, api_key), + required_inputs=AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS, + ), + ) + + +def azure_document_intelligence_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient +) -> tuple[OcrRecordingTarget, ...]: + api_key: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_API_KEY") + base_url: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") + if not api_key or not base_url: + return () + return ( + OcrRecordingTarget( + name="azure-document-intelligence", + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), + strategy=cast(SearchStrategy[OcrSdkInputBase], azure_document_intelligence_input_strategy()), + invocation=invoke_with_api_key(client, api_key), + required_inputs=AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS, + ), + ) diff --git a/tests/test_litellm/ocr/fixtures/base.py b/tests/test_litellm/ocr/fixtures/base.py new file mode 100644 index 00000000000..71a3298cd74 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/base.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import Field + +from tests.route_parity.fixture_models import ( + FixtureModel, + JsonSchemaDefinition, + JsonSchemaResponseFormat, + SdkInputBase, +) + +__all__ = ( + "DocumentUrlDocument", + "ImageUrlDocument", + "ImageUrlValue", + "JsonSchemaDefinition", + "JsonSchemaResponseFormat", + "OcrDocument", + "OcrSdkInputBase", +) + + +class OcrSdkInputBase(SdkInputBase): + fixture_only_fields = ("contract",) + + +class ImageUrlValue(FixtureModel): + url: str + detail: Literal["low", "auto", "high"] | None = None + + +class ImageUrlDocument(FixtureModel): + type: Literal["image_url"] + image_url: str | ImageUrlValue + + +class DocumentUrlDocument(FixtureModel): + type: Literal["document_url"] + document_url: str + document_name: str | None = None + + +OcrDocument = Annotated[ + ImageUrlDocument | DocumentUrlDocument, + Field(discriminator="type"), +] diff --git a/tests/test_litellm/ocr/fixtures/common.py b/tests/test_litellm/ocr/fixtures/common.py new file mode 100644 index 00000000000..3aa1ed61676 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/common.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from functools import cache +from typing import Final, Literal, Protocol + +from hypothesis import strategies as st +from hypothesis.strategies import SearchStrategy + +from tests.route_parity.fixtures.pipeline import RecordingTarget +from tests.route_parity.fixtures.media import dummy_image_url, structured_pdf_data_uri +from tests.test_litellm.ocr.fixtures.base import ( + DocumentUrlDocument, + ImageUrlDocument, + JsonSchemaDefinition, + JsonSchemaResponseFormat, + OcrSdkInputBase, +) + +OcrRecordingTarget = RecordingTarget[OcrSdkInputBase] + + +class OcrFixtureClient(Protocol): + def execute(self, api_base: str, api_key: str, case_input: OcrSdkInputBase) -> None: ... + + +class OcrSdkCall(Protocol): + def __call__(self, **kwargs: object) -> object: ... + + +@dataclass(frozen=True, slots=True) +class ApiKeyOcrInvocation: + client: OcrFixtureClient + api_key: str = field(repr=False) + + def execute(self, provider_url: str, case_input: OcrSdkInputBase) -> None: + self.client.execute(provider_url, self.api_key, case_input) + + +def image_document(text: str, font_size: int) -> ImageUrlDocument: + return ImageUrlDocument(type="image_url", image_url=dummy_image_url(text, font_size)) + + +def image_data_document(data_uri: str) -> ImageUrlDocument: + return ImageUrlDocument(type="image_url", image_url=data_uri) + + +@cache +def remote_pdf_document() -> DocumentUrlDocument: + return DocumentUrlDocument( + type="document_url", + document_url="https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf", + ) + + +@cache +def pdf_document() -> DocumentUrlDocument: + return DocumentUrlDocument(type="document_url", document_url=structured_pdf_data_uri()) + + +def document_transport_strategy(inline_image_data_uri: str) -> SearchStrategy[ImageUrlDocument | DocumentUrlDocument]: + transports: Final[tuple[Literal["remote_image", "inline_image", "remote_pdf", "inline_pdf"], ...]] = ( + "remote_image", + "inline_image", + "remote_pdf", + "inline_pdf", + ) + + def as_document( + transport: Literal["remote_image", "inline_image", "remote_pdf", "inline_pdf"], + ) -> ImageUrlDocument | DocumentUrlDocument: + if transport == "remote_image": + return image_document("invoice 123", 24) + if transport == "inline_image": + return image_data_document(inline_image_data_uri) + if transport == "remote_pdf": + return remote_pdf_document() + return pdf_document() + + return st.sampled_from(transports).map(as_document) + + +def annotation_format(name: str) -> JsonSchemaResponseFormat: + return JsonSchemaResponseFormat( + type="json_schema", + json_schema=JsonSchemaDefinition( + name=name, + description="Extract the visible document fields", + schema={ + "type": "object", + "properties": {"title": {"type": "string"}}, + "required": ["title"], + "additionalProperties": False, + }, + strict=True, + ), + ) + + +def invoke_with_api_key(client: OcrFixtureClient, api_key: str) -> ApiKeyOcrInvocation: + return ApiKeyOcrInvocation(client=client, api_key=api_key) diff --git a/tests/test_litellm/ocr/fixtures/config.py b/tests/test_litellm/ocr/fixtures/config.py new file mode 100644 index 00000000000..fe5cb566e36 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/config.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Final + +FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR" +DEFAULT_FIXTURE_DIRECTORY: Final = Path(__file__).with_name("data") + + +def configured_fixture_directory() -> Path: + configured: Final = os.environ.get(FIXTURE_DIR_ENV) + return Path(configured).expanduser() if configured is not None else DEFAULT_FIXTURE_DIRECTORY diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/0b234402bdfd3be223e731051202f2996578fff02d93ec3011031ce2b4c511fc.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/0b234402bdfd3be223e731051202f2996578fff02d93ec3011031ce2b4c511fc.yaml new file mode 100644 index 00000000000..f7aa8598016 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/0b234402bdfd3be223e731051202f2996578fff02d93ec3011031ce2b4c511fc.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"image_limit":1}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d3637dc2c090-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:15 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-5a87-7148-8ae4-4e14967bebf3 + x-envoy-upstream-service-time: + - '226' + x-kong-proxy-latency: + - '19' + x-kong-request-id: + - 01a05e89-5a87-7148-8ae4-4e14967bebf3 + x-kong-upstream-latency: + - '227' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '56' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:15.394028+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + image_limit: 1 + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/1f3001924ef2c46d3eccdedd808c7ded7fea981215a478a12d8636eff8eb1157.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/1f3001924ef2c46d3eccdedd808c7ded7fea981215a478a12d8636eff8eb1157.yaml new file mode 100644 index 00000000000..f06a1033691 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/1f3001924ef2c46d3eccdedd808c7ded7fea981215a478a12d8636eff8eb1157.yaml @@ -0,0 +1,60 @@ +interactions: +- request: + body: '{"model":"invalid-ocr-model-for-parity","document":{"type":"document_url","document_url":"data:application/pdf;base64,JVBERi0xLjQKJZOMi54gUmVwb3J0TGFiIEdlbmVyYXRlZCBQREYgZG9jdW1lbnQgKG9wZW5zb3VyY2UpCjEgMCBvYmoKPDwKL0YxIDIgMCBSIC9GMiAzIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PAovQmFzZUZvbnQgL0hlbHZldGljYSAvRW5jb2RpbmcgL1dpbkFuc2lFbmNvZGluZyAvTmFtZSAvRjEgL1N1YnR5cGUgL1R5cGUxIC9UeXBlIC9Gb250Cj4+CmVuZG9iagozIDAgb2JqCjw8Ci9CYXNlRm9udCAvSGVsdmV0aWNhLUJvbGQgL0VuY29kaW5nIC9XaW5BbnNpRW5jb2RpbmcgL05hbWUgL0YyIC9TdWJ0eXBlIC9UeXBlMSAvVHlwZSAvRm9udAo+PgplbmRvYmoKNCAwIG9iago8PAovQ29udGVudHMgMTggMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXQo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKNSAwIG9iago8PAovQml0c1BlckNvbXBvbmVudCA4IC9Db2xvclNwYWNlIC9EZXZpY2VSR0IgL0ZpbHRlciBbIC9BU0NJSTg1RGVjb2RlIC9GbGF0ZURlY29kZSBdIC9IZWlnaHQgMzIwIC9MZW5ndGggNTYxNSAvU3VidHlwZSAvSW1hZ2UgCiAgL1R5cGUgL1hPYmplY3QgL1dpZHRoIDMyMAo+PgpzdHJlYW0KR2IiL2w2I3BIcSVSZE1bNDNaKzdadVpbLUFpNUtMMzIvY0tJbkxtaixYPmxxXEVxWFItTGs6PWw+I1lObGclcFhybTBwMGRhSFMvXHBpPC5cPyFyU1Y1PGpoNFMhcj5xc1hSVXAtN18zVjItWWksZChNMkNrWk5yLz9aTUFIV0FwJUthNz8nWS4zRzs6WW4zZlZ0TVtFXERkPkY3b2FlZjgrbkw6XERhWFhyMjpMN25AWUAzJ2ZWWktaMTh0aDtXMiVJTV5yWylmQk08RyhULVx0VCNERidTRWktKCM4YTApUiVEODMyazdNQGksOjU1aT5GZTouckk9amVcNE9rW19gLG0oMEI6MW1ZNXE4RURtVWRdazs/KCVIbS1zcFAsMDhuQzlbWy5yST1qZ1U2ZTIsX2JeUz1eNXNhXEwzXGtqQ1RmZkMhWWRyKXBVVj1iSk9janByOG9FJ2ZWWTBETy1KKlg3MTZwTV5yWykyb0kuMDJjL0pGSDU5VDtFLExuNzlBRVk1J2c3SSpERSdUUVhqSSpvI0pUSHNtb2BuOF9tcFlHVS5ddF1sQ2toOGNmMGckYzsiLWBHa0ZpUSppIjUyaWVlXSMlQD5baXJCMzg6bzpFbzkyazY/XFk5IUNiRWhwdURNXnJbKTJiQD07bzY6KD5EMG0lImdpLi8mXVFXWjFoWFBwYy8/Wk1BZ2YvUE5fVl4+PzJYRlJPRVBvPz9ubGgtaVglMlIuWmFmIkxFcmtpOFNBLl0lRFVsKSdpZURNL25saSEscmFZXWU9XjVzYVxNLzpcZzBhJSVTK0o4N1xEXlxnXVUlS1otVU5UW2A+KD4lJF5UQ2JyJih0I0QiZklKP25QYmw5Q3BiXyRDQ0I5WS4zSEcmcFooWD9cOlFOMzVcWT1RKzI/TTM5U3Q/PjpJY1M9NWo6KW09WVQ0cC8uMmxZL0NNMW8lRiJvL0ViJlY+JDteb1dpO2omWCM1PjxjOC9tPlhjXVs9LGExPkRpOkRnZk8nRFYtcWA/UXI9O1pQXT5fcmpIW1pdNWxyJTBJLm42Q2VYaS0oPTlNWiFFJnJRQWdFayI4a1lmQ1Iyb2M+LlMlUTcrazlZL0NLX1NLQ3NibzMvLS5HX1A+KlErLWdzO2RNZT80bHU9Tj9aPzxsNG5IZV5ZSHN0M3BGR1o+WSZjQT9yVWd1V209KiJwYms5O1hyKj8+J2FvLilZUihHMjdub3BrOWdBO1YvMFIlWExqbVJBOC0yL3FYQC4pIztiMjopPE9uVUJvQC4pbjxha3IwdSFWXCw0QC4qYTxha3NsaG9Cbk9wWUlNIXFQQmlBLGlLXms6PWtyL3BfYiQnLDhdPFVQPWtxPFhvLDNiLFdWKVNPLzhlWzBhRTVPMjVmbl5DTXFhR0JxY3FmI3JeS245Yj9UbDNpIiNPbWNMMTNdViFvbnM4bURAP2BvZWdwUSt1TGlwUmBzVSFTZ2UmJGhOKGhyWjdCX0knMUJrRTVVY2hiXEoka0haQjpcaz1TdFoyYWNIPV1RVTNmZ0RBN1xZL0NLX1BOND0+Q1kvRXBHOSZqMDhtRD9UOllydCRHLWVxJlFkVGlVYCEoKllqWG5cbFFpIkNAMjtSQ2U+UT9dZWhTUXBLXlY8RFhlN3UvKSJgT001KD9XIjpycWBndGNXbDs3aSp0YiFiXEwvVEZeQSs0XElvXm9LdW1QZGYqOyc6XFp1XWQscWFfK1BkYD9NYzE5SnRJU1ddNjRCJDc5bmVVQXFFVzAuYm10cClzVy5eKHFba1AhW29baXFmOkZpLC8tLz5NSVFiVkxrUSw3PllRI0krTD5sSkAlNHBrNTY8NVBjTmxaLktRMCdDOFg+JCFCNW8mMDMyRWpdUUtsMmZCcT8wKjwsPyskXycmJSY/Y0JTUjJDIz4jbEwjTER1UEY5cloqL0taP1dqO3BQM1lOUVRJU20rQlhnNmJTaWNVUidjUV8mbTsyVT1MVHItZDpfbmRhXiZoYV1McD9OOW9rUihTV21ITjNwTzlwMF9ia048Xy5hKUxfQ28sQk1YaEJkWFIncVhKYCQhNUhmbidORCs9OGUrVkdgNS09Z3NtcGU0bXJURCxrJEsmK1VaUWJXLl9kJj9XWkZINl4tU1pvW2k3KUljL3RrU3BRVTc0Ri10dGY3czlScmk8VFQpai5sdEgvSkM0Jz5HWFA/RVg7XGo9YTxjcFxybVZuIypkXl02IkdPUVJENWxJcjNkMl9xckk/bTlma2BqUnI6YEs+KnVWM3BvSy9uczpgLWRwSiM2cGM9SS5cOWpkaT9fNG9aSWs9WFldMCVTJURSVjNJSDFFVGBDTjBcZ2pBPzZYVWhCTUhQQllQYSRXP2E4TyVRcDslRWkrTV1HcGlSS1lnZjg8NUxmZTxyVltOaD1LNllhL20kRy0vPVRTRDwqP0doSkljMEN0UDInPDVLKixwW05CUGsiUzouaj88JWIlUyFWLTdqWnE9R29wQ25AR1lmVzhfdWdhRDBQTCdIO3FgalBrWF51XSZhKmc4Yi9KTUNLb2lOZSJbT25YTHNUSWZJQU5ZOXAuSFI/WCJWaEwpdVYhX3JaO29icyREXUJUZyMqcm4/P1tHNFZZY1JqJTdcYjtSSiM3JEhbQG9bZCtKWkJbcCtnYSUuZT9iJk8xMFVhOCpYTTxtPWI8P3U0RTpgLj1JLj9kP0dWaSJLRS4/K2tTNHUqcWlNT2NwcSJkKjlTLEE5QG9vXkNlWyJDanBnaF82WGJOSi1DakRXQ2BHXWtSXGRrYUpNYTYlWW4yWWU8ZCxtdGNKUjxzXD1jSTksQWhwQ2dQSmRsU2dTcmwxVz9fNHBgcmBWKzM5NnBlJCtGZVwtIXJUSCQkS0hLUUI6JEE0bGdfT2UnRUdMZlEpI2JrPHBQK0tZKyYvRVdrQEFNZVR0TkJsZk0nUzBeN0xyckpUYF5pPSM1U3FtdT1ES0Q7JFIxSHRaKTkhJT1SJUBTT3FoSHFjUkRhO1hWbmllK0ZCOl9pczY5WWpdKVkzNmRiTjQuNyI0Jyohci5XQUksOFk8WGU9SmxPX1ZEP21qaUxgakRpKT5WNGFaN1ReRyVLOWhIR0YhJDdHTjVQWlJrJGNtM25bckFFcHRAbitlOW8/MUpwUk51ZVZJZ3I+cUdnRlshYj9scmwsXT0/RyVBZSNbY2ksYEQ/U0lvb0RdYUg8VTJYTWQ5O2UmaT9vMUkzPHJROjpfR05KIlNsPlc/ZDNPIkxxLm4oWkxcLzYhNk1WIVlQTjZURVlAVDIyaFhmVVdqPV8/RVAuLkAoUURYLltjZUdAXF5ML2xpaFI8RGVObCEvLFQ6QUZOSyRiaWg0OFdlIiQ/LFpkR1M5Zls3WyluTW1dVCYwXVt1Y04pPDxGZjlwMyEjXkQ8aj03YUU2InEkVFpVakxyRTtNYzlCZG1gQ2QwNEYobDhmLixWXkNaImxxXCw+QDBIXWMndEkxP15OKE9YIWdRUkQ2LXJZaydDaSlrOUBwRkNgPW5tOkxfam85UFdFLV06by50O2o4aFdWLlNyXVIxZ0gsOTUqSXNMVGtsRUI9STRsZGw2Xz1vSTBIMUtoWFk9OEpANV5EcFBtb1RNWmpMczhTTXUzTm47cm4yP1lQZE0oSGI3JC5uVHVzcWpUWFBGcjhsJF5eZ0IyIzRdREUsXz1vSTBIMUs4TWY2MldTLFh1LDByUThLPV1PVWxJcCMhWHFQOEFWXW5zOmBtLF8nW21IT3U7MEsndWU/byZMaFJZUEFrOmI2dSRrcTkrdEdUImpEYShiQD84SWNXNFloTkBma0pcbWM5SUBtJ0NTa10zRW5DUClfXixrX1BpJnBVSW5vb3UrW10mXEddUlFkP1JISVguYWExVmYxITsiYW9lQF1DbHVaRHFmKGVSLlA0c21tNkZDLk9tLUU3Im4kJG9aN0luK1xWPDsyS3MnRDtrJlBcRjRZZU5qJUFkPVs7P28xSTNXVyRlVlMhbVheSyRiaWg4LEJoTERkSWhlJFYkP0BTYGZIMzRXYm5dMl4hVWRCVGckR3I6Uy8uXXRhdXU/U0lvb21qKkJHOSJvPi4kSGxkMVJPJT49SiI4QG9pNS0xb15ML2xpaEFRPWgyPmtfPC1uS2thQlRnIjFyKjlzKU9VRnVmcmRyTkNII05HJDxQPy9XZmE+bzAwVWE5VWVjJ0dXcFAsMDtZbCJbIjhiMWEobiIrRmNcVG5QKj9TSW9vbiJwIyRNdFc6JlVwMGIjVUlrJTVrak9AOypjP19MbmVbNjNhYTlTbkxONURjaTxtYjgwVWE5VT8yZ0YyamxhVzUvOllLTFAuVlYsKV48Y0UyYypyWSRWJD9AXXVBa1dRIz5RUEQjKTg8cEJsQ2NoWnJdP3FoSkRUbyZUMyxEaG1OJm8vIkhwMFVhOCpGKzddXTRtRGUzLGs7YjEtSFkjcWhmKmlPSFRqSE5xdXJpZW8hXDNNNmYuRitbbmU8Il8uXSJFNG9KKjwybCxXTVpjUVEuUk51Zj1KK01gXGw/VkVDbmVbNjNha1AodDwtXFJtPzlQNGEiZiJaW1tkbEBdaFVuRWQ6K0xoMUJUZyNWcXV0LlxoUiE3K25lWzYzYWtPYUBONF4jazQmN0pZXy5dIkUqVzFXNVMoK2UtcjoxWV8tSFRLWzAyZ1o+U2owKjZeTC9saWhIR0VGbnR1LUk8I0V1YXI4bCUpa1YkOSFtJS0/SSIxamM0NEYrIiovV2dqckYmZkM7QkFmMWpQSj9vTDxdSy9qVCQlazdeSD9mM3JtbSlnVldaWmxucz0pUz84bWZINlxBSEtkX1cmI3JFY19vaylvQ2I8VFA2LlhPY15TPlc4LmJobEEiWzhbbi4pYW4qQGNqUz86WUgqRl9pSS5nJ1tAPG5vInJkY0U8JGgxV2RdTiUrSSlqJWZzSCpkSGQtI1s1PGE8J0NxOllvJyM0XHBSTzRpOyR0bC5QLTljbGRWcFY8U3FxWWwqVilTKWdiTz9FMFQqJEhsZUpdR15dcDdeVXNHNj5qNktuczpfQlxBLT5KbSwxX2EyY1g7XmJFRm05P2lPNGRuWWhQPVZtLSpcYVcnSTRkUW1OclcwIUxFWWwqVikpcy1QUi4oOF1GT0lNckpyOGwkPmlwWGt1RzRWWW8pbTZGQ2ppTWw7bikhZ3NvVmRpalgwRE5gYWE8Nz9wPkdXOVcvcXNzWWwqVilIZkZNOWFMMV0+TztrJSNyOGwmNG1JMCM8Rm5WYnEpa3NTN2ppUC10bitpPi5wU2ExI2hRP187YWgtTU1hRXA9KU5zYEZRJUVpK01dVzskJDRlbDxBOydHXDBCREtgN0dLSl9qSCMqJC5fVksqNnJwby5DcVxwciVEO0wyXEhfJj5lVCJJZ0pBKidpO0hQXm4kSDI1WElmQWdlWl1YPCRnajAmKS9Jc0xVNm9KXS5vZzpUQGJRWW8rZ28mUDVmRGolY0BHcVw6VW1tQCRFakYtJyxUIk1JVm4pWlRLNy9sYzxyUUYodEgjKiJYYzFUa01yOGwmdGtcbW9CWzxaXVJEVldqb0gyI1oyR3JQWjc0bXJCUnI7REV1YWgtaixKQG1cSm8oUDs5LGskbzIxXC1SYmIyNS1rUC5aNm5hNWlmMFErcWdCOGIrcUBTV249OD04a0NYOGIrTTRyMmI+MmsqXWosbz0nK1NablwlXFNyTGtMbnRGSyRtZF1vSE1Dc0goXGBNa0lOU2ZCT3EpPFdoLk5dcidXYS5jNj4sU0VLS1g+Tj1vYC9hbzAzcDNKZE9lUSFkJTNsKkdjZW1TMTM3NyJcdUtENHF0WDU2R3FqVGAvXEgqISluVD1yJGNqamRwVEQpT3EmP2gsOklDPzpNWWtPSjstPGZlRDxGRCxBK0dOU1J1Oj9NcmU+N2hrRSVMak1ZZTIrWEtfVWNoanU+S1QuPW5pPmJvNy1wTWAySVgyQTIhJGZmKTxRRzRnJUluYFoiOG1FMkxVVGpbUGhGZVs3QjxLMSxFPC1FaWhuP15MMjZIIjU+U29EKEBKUk8/Ml9Mcl0+X3JqS1ZDRi1IPTgxUW8xbjYoOUBHRXUham1VWFNeNS9BJSxxYi5HVXAwZGltdW84ODQnMUZZXC8+WHM/ZnNVKF5ba11lJGZjc2wvOlRoW11fOksvLHFiLHFYMEROQCddKz5CYC5gMTUncGNWJVdpcWQ/RUZJWVdrSzddM0dKTHNAYEdOZjNxV0JuVG4qYzR1J1FIM1VuLDZNMUhNXWVkRiRhbDlFdDlXbXJlPU0uV2I2PktGJjFwbXJXXFxBWVwuc1ppLDI+XXIhZlZbcWdRTCljZVknUnEwUTNkbSFjbm9jZVhzUnE3Qm9RcmRNZyxjWCEpL3E3QkwtaVZlV3FUIj9oPW5lV0VlQEYqcmlhUyFrKVxcQXUsJFtXWmA7UWInclJKaGldPk0tOWxwTXJGZzhhQSQtaFNQbCddcGVoPDhpY2hSYkY8XmppZjwxXydhRWovM2UtXGpnO0xONyhXUStyJSppV1dyZyhXQDZnJSdIKHE0NnQ1PVMqVCRWUWpKPTVxKixoJVUoZj5UODFePl9yaktbU2xsLlxHVnVaQmtfXXJLVGYoSD1fbU4kUlYvWSk9c1UvcDwwS0V0QC5LMnVfXyVNSm0oREVcSXRxVSYucmYiMF9haCFXVVx0cG5BP05rKW1EOm0wYk5qRjchO0ZccT01cSosPD0xLDhGWyJPPkhQVF08LF04SVUpMHUxUWI+SSohXER0LEVLdTxNLiwqX2dRWS4uJi5ZZW1ZalNWUVtcWmFmIUFFS0xTNWs0S2guOF5IQ1JTJkMiMCowa2E0XmpVTUU9NTUmUjdjQkhBWCUuJFdaYWYhQUU8LUU9a004Ky5cIjpQUjJpNCQrbjZ0MWVqIy0xKz5dR3Q7Vz0jOVloKityZVQjREYnTTQ6RjRHMkFFLVtJJSslRUtkbigkTGQoPjRBPXEwYD4oPWdTIl09QGhYYjU2bW9gbjhQSVRRWVlOPzlTNWxbRnAnZlVzZ1g3MCRELi5lNjtNXnJaWDJjL0lySXJGLGpaYWYhQUVGW2w1QENZV0woMEI5OkRXJTk8IS1FLyppZWo1RE5WWSVfJ2V1QG9FUShaMypyPlEkUmolNnM0YzpoLicpMD5NMnIjWyhuKk83X0VLZG9lWTVRPmFQUD1ANmA+KD1nXTRdbkQyY1xfaEhQVF08LGpwRDJeVWAlZyVkNyYjRUtkbzVwQU5uTmpJT10pXERlaVRRUTFEN0BjU1FZWmFmIUFFVyU2YF9oSDJvV0ZwNGk6I1wzXnI4bzNiKlMuUixpWy9ZLjxvVzUzZjxgIUgtPzI4XU0zaUVzY2YwaE9gOWVvP2U9T1ZvMnNMIT5hWkkiRkVRKForSUg2c1RaYWYhQW0zX0NhbW9gbjhQUC9vcHJ0Iy1rPW0jfj5lbmRzdHJlYW0KZW5kb2JqCjYgMCBvYmoKPDwKL0JpdHNQZXJDb21wb25lbnQgOCAvQ29sb3JTcGFjZSAvRGV2aWNlUkdCIC9GaWx0ZXIgWyAvQVNDSUk4NURlY29kZSAvRmxhdGVEZWNvZGUgXSAvSGVpZ2h0IDMyMCAvTGVuZ3RoIDYzMTEgL1N1YnR5cGUgL0ltYWdlIAogIC9UeXBlIC9YT2JqZWN0IC9XaWR0aCAzNjAKPj4Kc3RyZWFtCkdiIi9sOTYpO2cmT2BoPmVYVDgrZFFNWkZQbm1RMU04ai8kJDM7ODVOYzljbTkjSVk2JWc2MnBGJS5ybT9YTmoycWQyU1I5MXFvTmxnT01pJyNVYStVUiUyXWQnPzQmOkhlYVJEZDdfWi1eNWdYXERlUXFoJiRJbEJcWzY/YEdOTktQPUQnJ10oVDkzRVNHRy5VXzBEX1FAME5NZ2JyRVwvJ3JaZG9dcmluaWpSZkxEIixdbzwoRjhZTV5yWmFZMGQ+OVxTOj9tZTdXZD9EOFdiI2kibj1CKkd0WChZLi8+WTk2XkUkUyJQY19JMjVvPipvKWA2UzNxMmRTJ3AnZy8/WkwiXDlqU0AqQi1xPD04LGomRVNHSEFZUEFFQFMzJCoqWmFmIydpTFg8RGNlS1hjPmhXaDFgR05MVV1bbXElXS9qaXJaYWYjJ2lMXHE/LE0tITtmQ2A5T0VTR0c8bV1yRExXL2VMZC8/WkwiXDZjWGhNPGJQX1U8aGhWZ09uaGhZMHUtLXA0am1eNSlVcS8pbTkocWk3USt0XidGUiJNXnJaYVkuSE5eL1Q3MEoiTUUoP01WRWspK2NVLWQpaSs7Vm1eJ0ZfW2MnW21qIVQwTWoqTV90bUJhPV5bYyw0dGojUWhRaFkmWiMvP1pMIlw9cT5xNE03cig4XGROSi5ySDtEWm5yZ09EbGRna1hqPyFhZ09nSTsoQSZYJGpmQGBwYHAnaD5palJnQzhgT2dgTDQvNS1JMjVvPjJWXnU/NFBucEttTjguS2llbEtGUmswT0heYDFVVzNuVl9aPl1KVDFhaT0rNE5xZDRFM19LNiJZLi8+WS0uNyNeXiUoMV9cIillbldUSGguYG5cSC4ncUguNSpHZmRDRElbU09idD8nOFVDTzw3JyI5QU1zIjNISGE6KFdeUl5qJz8wYzArKU9sVSlALnJjTUwmTURKV2E+cD9tZUV1RWtnckEuP2xnbCNUJTlLJTRwS1dSMFIjPkk1P11kLnJaakElamRhSCE7XGhEcjZYakdNJS9EWk90OStCOnNxUmpXNDhDYlVUbmwmYjw/PypxQVFLOmhLPFNmLCVeJCc3KDFaK0Y/Uk9gQHNGLGNOPGZjMFc7IU02TClpanMzdWY3RF1yOT4vMiY/WkYmLWs6VSRDSCI0NmxqbFwuc1xGLy8mcTNPJUs+Pz5qWmllbSdJaDNgUzhqMDQqJms6PF5GbWJHTERRVCYxJ1F1S2UhTVgvUkRTcCNDc1onZlVwKWsnaTU2TE5lbzRhQTlIWCNQTUM9NTtDZyspY0VMWS4iW2IoMEI5dWpZR0taN1dhRSRxXmVuWzVLZGk8amMvLEYoMEI7W0NATSxsRiw7aWcpTGZZMU07K0VQaipmYSlQckkiQ05eJFYpMldqai5VRzIuWFJsOmhIJW1GKFMyV2hTRFVHMi5YVGY1YCwhLXRgY21FMFIhWW4qI2xcXk1NUEI3VydFaVxvcj04WVwhLkYwLkNwOic+Tz1gLDNeLVB0IzM9a0BTWEtTLiIlVU07K0M6L1dJV1tjZk8iYzRXVUszJyI5U1NnKS41Q1REUT0sRkErWjgtI1I0MU5caCs3NSRiZzA0PWxUKSciOVNTOWNBXzJUMTdcQSojWlV1TTsrRGUuPzIzVzpHNldLQXAoSyYua1YpZGlAUUhRaFBabydUJ0g+dEZhZFk6YklMVXQtWkFicVJRQipoJkpfY29mXTNNOUpsMmooNGVncmVOZ1tObDspbWshXE9UV1NvJyk7cmlpO29xUE9qWzRcT281VTVrWDtsXDxoNlotYFhXJCV1Myo4bEBXdShPL1RRMmRPPUpSNmZYMChvWEs4Kiw5QUpfKW1lNl41PjlwWXEjM0loXDtsN2lOUFtrREFEWykqcXJkZ29vZFwvO0ZOU0FlUE47WkVidVtFTUhVUGgxOHUjKmU0LjdHYDJgRmtbamEiaSQwJEJjLGIuRU5YPmZKdXJadFBkUTZ1MDpgXk5jXFZfbD1iYVhpYVgtSV09P2ZIPkZgP2FgSCxzRFlYakRtT0leTzouKDYrclAxISw3RmB1NGBCPD1WOGIuPm1RNEtidV5Ga3U8Ij5pXiwuQ1hbVTxRSz09NEBjLzxucDVTVjlqVmk+UGU9OFYtSFlUS3FhUVhaaV1cRT1JVEtpRzFlRCJERF8uamkyUXNXOjosdFYoalJyMiM+b2cucGFnbGA0RUpQc2BsZz40VzNhaFItRV00SjNoS1ZKUWpQQj1GbjorXU4jQWReQVk0NGQ9TkBDO0FuIilJbXM4LnM/QC5QIk9tNy9VRDlbUCwyXCVPTm85OkRwSVw0X1VVbzo7VUludCgpOXBmRUhNImUxajldNlxZa2BZOy4xcFdzbEpJVi9FZ2ZYcz8lTkRsJCJWSikuKjAmQEhFZl1RJkhqc0IhU3JLXGpNOVtQLDJddD8pdGFVXUpIYy0qb1lQSyoqRT02LlVvUE42bFlgcGVjQEkyNF9aRHNtN0o8bU1SRURhJ1U4WUhLbT41PG8mTWZZITNTTFhJI2ciKjM2MjFlRCJ0RFtcbCQ+LU46YUVtTFI6NEJePUxha1REQ3BxJkddQ1xYbFFsOU5QSVAzWDxPaU5aa1NQckhqckhOUTIoaSh0Ry1IMkRTZU46SD0wIVY9R1EjaShVdSloLl9HZ0RdZkhXLW5JcURiU1AoSCkmJi1yLltPXF8iS2RfZ2NJMC05W1AsMic+LSd0T3QoVWlwPVssJ0g3aiwwZDNPIVldSjk+VSZwOyJgZnMuRVZHISYwR11FbTpfaFk9XSYwNmw4NzJzQFNMS2YzKWpSJWQxQ0FwUnReRDk7J25tQl1HU1tvX0ZSVS5PbyZIWGxpKVJIZj1pbzZEVz08bzA0PDhiMiJqQSwpVS9sQEE4Q0ZBJTY3Q2pobkw4Yi8wbGo3My0zT1cqbGZVbydeVGlsVzM9MWVIT2pEajcxJlthM10+T2JVTjJMKmsjSGxdJDo3XyJOJldONyplSGRhMDdiYklvVnVgMk5qSWNgOHA/Zy01VzxeJmlSaU1DLUgnaS9KbStMUHNYRVY2W1InaEZMLCJqOT4obiYvOVxNbUJhS2wpQzk7SVVJayU5R1tuZzhWby81XT1LPSQ6MmoyUTZIKGhsKmA7KzlOW2ExOG5rYCEwOk80Ty5HclZfYGhCYDkoIiguKGY9JjZkKFIyKEFtYCJEPDotS0RaRmptJlApKTVWPl8qX1dfMGkmX0lvQUY0bGZEcWcpWy5uJj8lPlYxJTNZNU9FdT4/OUErZzUhLEsxbnEpM3VcOktocV89Zmw7XEBkcl8tSFYyN05TdHFlRzZRYWFUWkhbbjE3QktXKCNCZEYiNXQ6KWpiPEInXUoyaj1qTHJNT19rISVsaEIlZzJuWSsvaVlJbFNZUE45RXUpdDNMLFpwISYqXTsxQTEuclMsLyxeSFRiYGMrVEUqUSVPRzBOViJVSCIyXllhYTlkQUw6YiRGU2cxJmlUIXEiWFVYOj80MEQzPihBIVZORXAzQFM7ViElOzwxN0JLV14ic1VvRDdXclk1KTJiSjRnMkxRNz9jZkdnKUlyc0lHUVlXVDsvN0FTbGFkOVUuT3E0bVokMDg1Tzk4Y2NPJiQ9Y1JXcC1kM051VkdbbjgqR008PmRGK1EsZXFPR0kqYi5FM09ha09dUG4kYCkoRitjYFQ4XSpdcF1KMmo9alIpI2ZpR08tRkgoUW1oaVNKM21AdTtWVGdXVVZTJTFndWRkY1kuVFxHJy5fUiVkMUNwQEg6Q2NgY1YpbWRrTm4mXmtYZVpiLyJdUE42VEZOVCU8ZW1PP2slakdkXCw4YCRXLCkiLCUpKk9YWGAyajJRNkdvTSo/XEc5WlhQXSZoRDg0SmcobUhvbmxybVBhO19KX08sN1BeRFhEZ1FFODkrY1QrJloiQ0JCNmUhRS1IU3A/TkZAcFlNW1VkPXBISGI+O2AlJ1AqY0RJbjZxInFUTDNvUVNOQEM7QTA5bkpORT8qRyVeSD5YO21YOkJbVkopLipDdTRxQnInUXMjKShmMidHZVNxTiFSRmooNWlEQS03WUVnXWRKdURlX2pIaFMuclVZbj5DKzFnKkhsLHNUJ29TISljSlJiPzxWN1VVTVlaQUlLTXBubzkpJjBCVGckUGhLVGpgVCVFc0VTQGczcURiU1AoSDAoI283U0RnPTpJNCp1L3BVcmEtSFJjVl1TbFNHcCQwa3Vqa1ZHNEB1O1ZULnJUZiszWGU0TlohaU9qbzU5ZiowaCU7ZVByQ21eaDVIOCZZKlFeVDc/Y2V0Mmw6bEhWMUcuXFdTJGNaVExuXnMpalxSI2RLJVUvMDZxJmItSFJjVihmUD5saiwxTGhPdUZxKlhmcCw6PSc8NTNdT2Q8c29ldUNXRio9W143NlcyYWBtNydwP2QkME43bigxMnIqJT10TkpsbC5Dcyg8KiVEQ1pvZCRFTzYycGBncWpVRmxERipmJW1bP25OXDc/Y2V0Mmw6aytlLGMlX1dTJGNaVExuXnMpa1kjb1c8NHNGQzw3RG4xZUQjIWokVjcnbjMobW9rMXFHMkB1O1ZULnJXKFlxKVMtNWNfS1VYLnNZV14tSFJjViRvOycyMDc9MFdtcmxlbDtgJSdQSCIwVmdYLlJBUUhXJ0ZTJ01YYVdmYyw5Vi5dMy5eSUtNcG5vOSkmMEJUZyQ4aEplVXNdLCYoXEghWmhVUDtmQFBQLls9QUUwU0FTMCJrMGdnQSlmMGAmcUNQVVJeKWIpdCkuIWY7UEIoZCpEQT4sTi9DPylbUDUyaE08az9yUURtYz00THIvQHU7VlQ5QkQvJzkrQF4tR3A9MiRQKmEzLlJGYDxIKlYpWlRSTnNQQzJ0QVVXTGNYVVZBMkhtN3A+LjNGa0NCMzhHcD0yJFAuLm5aJERkZC8hSTVuLTlwMnVqMnAxUz9fRF5KN0EySG03WTIwL0A3ZXBaV05GWGs7aE5tI2M/SUphOD0vZVJEODRKZyhtSDRxOjFvZ0RCWTc+aCYpLDY6J1I1XilRLi11I1wzb2w5MjFlSFAhaFFVNlZLS2ElSzZmKEAnUE47LGklSyJsXk8/VFNBNmYoQCdQTjZUPSVEK0hpcVJqMVBLS11xQFNBVkpPXTdEPE5IYjI0O0RdX3JOJ0BcMUQsYm1zJT9MPGAzSFxeMjhCUV9COGFfNmpNcE5nPzxcY01MYjwjdE8hMTdCS1dWO0JrPUJHXj9wVj0haEMxN0JLV05TWlkqQF4oOEViYSJBbmhPKCQvbm06TD1fb2k1PEI6IV9mSDAoIzdSTyU+amhKZGBOIThELjViaWg1RG1IcmFvI0dGYFtoVm50XEJkIS4xLC02LHRIIjJeWWFhOF8jQC4tcmg/SDouWDdQXkRYMD45QGJWMUlGYlRdI0IxMTdCSUFcMlBDZSFUKiJRcHNUX28pdTQwTFlMM0M9XStBOFVQTTFgPiNNTEVMPiRZLidKb2InKyguXkMqOkkzMTw4aychRlJRckhYVCcuO29Ncl9FYmF1dDZOWWYmXFNVOysoXVFKMVBdRGROPEQ+aSJRVjFlRCIlRGFbNSNcXi85IUQzO0puTiRjWilqLEYxVStTWFBxOTkjKW5QMGtQIzYraT0pJUhLXStRK15KbzdQXkRYMDdsLSdNdWhldWVNPS1saEZMLCJqPVUqcVZET0QzXU48U01hVyY/OFAuXDNaODxzSSNUPTwiQk5aXSo1VDhkTkldUzU0YFFOaEsmOldmRGwrOnFQRi9TTDxbLS9MPT5Sb0F1P1x0cVwsW1FbTDcsXkshcDsoKW43NStibkNBKiZcIVJOdWdXMmVvMT9naDJ0MkRRZ10tNz9jZldvTD9BSk9TZUJFVGhVaDNWSikuKkMjMTZzIydjYVwubyUsVWBZOy4xMm9jaltkUW02LEhVW1IzaE8oJC9uZzhTYDdpS0hzTltYWUliOFhBOjhgIi0hNVkyJ2djVSRbSUlzR09dLUhSNGklRWhbLkAsPz87N3MybGw7YCUpJitFJipuIUxVPkJmNV9qWGEsNjlkaEMuUSklJSg8PUo5ZWJJPy46JDk5W1AsMmg3WzZlOlk8PT5cYGA3PjJsOjhPXVk8NyxPIVNXIWhWOkpAYTtcJjc4Uj1vKi8uRWtAPzNdaSszb2w4TFJPIyknMmNgRVwyRDVBXDpFQiNHJ01YYkJgPl1hYCNiLjA5XTVfVFZgWTsuMXBXc2w2Y2NgUDxXT1QhQS1aO10rLF5FMXVZcTZGR11QUUIiNmUocCxkM08iSF1sPjUqakxnLk5xYmJLVDtgJSkmNGA7PDQiJCYiXmY2RXRaYFk7LjFHSmJBVGBXND1YYk0+LWtIL2ZXNlAxU00vXCojKCNhcj8rTFAuXE5jQlRROyNIL2YkJVAxUykjNG49TC1HVD9RY0dvYjlYSCtnbDwtRWpxLzkmQTJmUE42bGFJNyRHKmY/SkBuUk8lPzdEWVtEN25FWmZvOi1iJChyUDEhLDhDXTs3YEI8PVY4YjBVVVFCLmQqWG9iNlk5SmwsZCslIk07PEJbY29RKWY8MTw7MEdkbjdwSyklPk5Ic1YkWnQxOD1uIz1cWCNIUzNvcSojXS5BJE8uSVJuPCxdQU1yYjRDTDs9Uz9CbipkUi1kUC5XKmhXNkhpcTksWjotLUVlIlVQLipndEttQ1hbWEsoNFttLjgtVU5QXCYkSCZpNkFCXjFbPFU5IiF0XUE9Yy0vcmkhUk5SMC4+a2hjJmlPQlhuR1s9KkswXVJiJT9tSShXMVFbLGEqZiYiMmYpSi1FS0tcWi5fT3VNLyg3aUZDNWBAXiciTmdbUT0+PldhPWE9RC1PW3JDIVwlRUNmITdXX3NWYklMVXQtWj03W29fPW06J2MyTy1VKDJJZFYyV2k5UD8yIUdEUik9YU5Vc0ZxPW51PFpnUmFuNiVYSSo9Ti8qKkA9NXBbT1ZQdXI+ZldNKDY4cyRScltsbFo/KlYuM1ZYaEtiMlI0RTozRXRkRHBhcjs7LUVAdCgvLSNTJ0lab1s5LWVfMyhzWygtZ1k0Wj89XG8jPVJFKnNDTSxNOytVQFZbZSFpPzcvNm4zRVAycUUkTmpVRF0lZkVxOGRWJ0RMOE91M29nYENTW1RWZVkvRzInb2k2ZFtFUywsPF9zdW5eaCJGdUNGaVlMTElDP1ddTVJhQS80bm5ZNyM5UzIvcHFEWyNBWmlzZ2NaTCtBR3BCSyQjXm1JaWd0K1MvRGc+aWZYKGdhZFA/MiFVbVwpXSZOUmBKKVQ1U2dYRUxGQWxkazVSS1puX0UpYD4oPWZdL1Q/UGBhSnR1YzwiN2pLPURrYTA1anRoRnNyP2U1KVVxb1YsbGJkaEZOX3RAQHRMUmYydFpeSl9WMilkTz1JZ1xEbDVjSU49ZmhPWD8oKTUpVXJaO0FoK2tYWitbZSZUa2JITUFzTEFXMiYsZS0tUC1cL001OEU0N1swZCteLyc4VjQ3JWhlLVtjWnFndUw3Qm9iXU5EYUFIOTZjQmQhSlFtQTc1IipcKWVPbms5NF1nWE9rRV1KSWY+bjdAS1tkXXU5JjAsRktNMXFuMS5nP3JGIVleTjAlPG4kOXB1WGJCQHUnXi0jUyZeSmldWHRRM1ZkSGdPPzxEQWArY0YuVnAhaT8hKjkjJyI6JDVuOiZAPFNWSCxhP1pGJG9pYDwkJywtVSwoTXNZbzInZlVxUVUoNnExRyVuMU9ZSU5RckVWJihQcVc5WydeViVuUE1WRilaWDpFOmBwNzNIIz1eNXM5RUFRLyhRJDZyWytGUCMsZ2gsPXVGaCRMPGgndTBsYD4saigyYUpKSUdySVg8ZW45IUFEOHFROWEnbWk8bS9jb2A+XUg2UTZlL0g0cForbVFaZDdXZ2dQZClMTmtOKVxZP3RdNy8/Wk0tXDNZJSoydEVSLyJNKWs8TVZGKC9Ub1JbOS9wZFRlWmFmIVFqMTs6JC5CYU8rXlgobnNnUGtJISVTKEVDNSlVcS81SGFHc3JGcmUtcUNKZVpjKyNCTWZ0bjJFV0goWShgPixqImZCWDE1cEBpSnJqMk85QC5ySTAyaVAjNislP2VgLUkyNW8+QiYmUlhyZi5wRk5JMU9SWS4yYGRnWGdobyVAK3JgSTI1bz5CJigpQ3JsUFstN1k8bV8uckkwMkwsc2QnJztYJEtjZjBpOk8xVE4uSXJdVitkQUc3TE1WRikqMm8lQDI5Y0JUIS8/Wk0tXDQuIVZlYnIuakhoOkhHRUxVciIyVFY0U15IUlErRERTRmEkZCRoMWZrNTxERDhvIy43YjloXi5ySS1xTlxQa1M9XjVzOUVMM0trVjU6I2JwJkZdYk0jUkpsM1NKSH4+ZW5kc3RyZWFtCmVuZG9iago3IDAgb2JqCjw8Ci9CaXRzUGVyQ29tcG9uZW50IDggL0NvbG9yU3BhY2UgL0RldmljZVJHQiAvRmlsdGVyIFsgL0FTQ0lJODVEZWNvZGUgL0ZsYXRlRGVjb2RlIF0gL0hlaWdodCAxMjAgL0xlbmd0aCAxMTI3IC9TdWJ0eXBlIC9JbWFnZSAKICAvVHlwZSAvWE9iamVjdCAvV2lkdGggMTIwCj4+CnN0cmVhbQpHYiIwUzk5LE4tJi1VQFxFIXAkZ3BWMjVzMGFrMXRuNyYxL2xDZkBjJmMmSVdZbGw5QUpWaz88RT8nSVdzI003Qj9BUzlPSD9qJUZDcWg3OmhvRkArSiVYZ2xMUk9Cay5uPTJNJXNITUo7bFFaVWdRNl4xLEZvb0BIMUJEVVM6dUdyIkIsaFBRS1ojJixIMyVOJGYwKz1ARlNlV2peTi89MSI4Ikw8RTs+KiIvbEdhQ1IkTW5nIW5pSyU2Z0QpV1JTJkBEKSsmXDRSMForQXM8KnMmcDcsY08yYlpeSz1baGFkbTktKVZmKFxhKEQsXzxXSSRXbUFzX1peVz5kam1WSDJlZjZnSGhPP2BpOExnVlRBYDxON1o7STIiLm1QPFNzIVguMiYpNDJbMypjSFwyMVNKZktpQjI+QTBjNlFxdVphajE0Qi0rQTlmaUc7TDFETGVlQ2FgSDJSPm0vR2VsRkJhKi5TSD5RNSZiMz9scSxnRFdyXTBPbD1UTFcuPzkoazhEY08mPF8qWUM6RkpvXE0jZ20sMkpZRDw1RDJrbXJLcj1fOU9kYmwiKEBZO1hLQFMpUHJsVlFNKEQ7XSo8MWYjQHUlXFR1Mi9wQ2wrPScxMy0xaTwwRl5UTEgxLGUkV0A2ayM+aVY0NU1LPkhGN1BGPW84XCNBOyttIVBGLCFVJE9WYCwqaTFRMUhZKCFpYGB1LjJxXVlpYVM6MjlVNGdZIzkmXG5YPV1XM3RmLEIqNF5gclNQbihDPjEpQVNbSHJPalIlbyZJTjNNbXFsKG1aREooPylFN2pSRVRwPzUkU14jWmFAQ05RUEgsJVtQSXI9NUg2QyckOkIqZSZRZTk0OyloY2pyMSdzZD4hbTxtcUYjO2Y3cUIsZEU4cjlcO2MxTC4+WUQ5MUZIKE8sUXFYNyEzdFpRRT1GXXJYIXM9XC02JzFEWCteXW9GbE49XGNZZDFDbVc4czF1dXNBNiRjT2s4N19pRmcqWjM0Y1hmdUYzYSplPzo+Jkw7TTJJRDlOUyxgYmwvVGQtXlFFS21NcGFeVGhoaDQoaWpyIW9fNW9NZEU9UEBXdTxrNTM/PitCMjtDM19XJFhDUTM/PiJLMjorQFNWKHNwTj1dTGpjczNdbDIjTkJ0JCdQRDo1O1UuI0hyY25yIVgjVURdNDZPJCUvKlAzU3A8NzspPG4hSS9ENF0/WFdON246Zyw/LkA5UCVpTFs6KkJrTlQvNGxDQC9acSFxaSxDZT5kZGtkNExsbmUxJl9RTV9mPjtQbDdWRTFNWldKbXInamxMcW1MUSo0X2lOaHItYmNKX1xjLldIIitwNkxPMSsxJjcnVzZaMjotUUZZNTksJ1M3PnE+QUhKLDdlYE8uKSJeSCtPPiNYPitLX0VPTjA+XWAqVHVfYUhOLlVEVnIhZWFNWE8lZUZDV09TMGNcVGE8UjdIZS9kXkhiVW5OMmZcJk9HSUdWb3QsMDhmODA8K3NSSzJtXkZ+PmVuZHN0cmVhbQplbmRvYmoKOCAwIG9iago8PAovQ29udGVudHMgMTkgMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXSAvWE9iamVjdCA8PAovRm9ybVhvYi4yYTg4YWM2NzEyYzljYTcyN2U0YWQ4ZjI1YmNiZGFlMiA3IDAgUiAvRm9ybVhvYi42YmU0MGVlYWFmMGQ4MWQxODc1MGRjMGE3ZjlkYzIxMyA1IDAgUiAvRm9ybVhvYi5lNWMzMTQ3ZDE4Zjk4NjdhNGYxMDhkM2E1NWEyNDhiNCA2IDAgUgo+Pgo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKOSAwIG9iago8PAovQSA8PAovUyAvVVJJIC9UeXBlIC9BY3Rpb24gL1VSSSAoaHR0cHM6Ly9leGFtcGxlLmNvbS9pbnZvaWNlcy9JTlYtMjA0OCkKPj4gL0JvcmRlciBbIDAgMCAwIF0gL1JlY3QgWyA0NSA1NzUgMzAwIDU5MCBdIC9TdWJ0eXBlIC9MaW5rIC9UeXBlIC9Bbm5vdAo+PgplbmRvYmoKMTAgMCBvYmoKPDwKL0MgWyAuODMgLjg5IC45NSBdIC9Db250ZW50cyAoVG90YWwgaGlnaGxpZ2h0ZWQgZm9yIHJldmlldykgL1F1YWRQb2ludHMgWyA0MCA1NzkgNTQwIDU3OSA0MCA1NTUgNTQwIDU1NSBdIC9SZWN0IFsgNDAgNTU1IDU0MCA1NzkgXSAvU3VidHlwZSAvSGlnaGxpZ2h0IC9UeXBlIC9Bbm5vdAo+PgplbmRvYmoKMTEgMCBvYmoKPDwKL0MgWyAwIDAgMCBdIC9Db250ZW50cyAoVmVyaWZ5IHRoZSBoaWdobGlnaHRlZCB0b3RhbCkgL1F1YWRQb2ludHMgWyA1MjAgNTI1IDU0MCA1MjUgNTIwIDU0NSA1NDAgNTQ1IF0gL1JlY3QgWyA1MjAgNTI1IDU0MCA1NDUgXSAvU3VidHlwZSAvVGV4dCAvVHlwZSAvQW5ub3QKPj4KZW5kb2JqCjEyIDAgb2JqCjw8Ci9Bbm5vdHMgWyA5IDAgUiAxMCAwIFIgMTEgMCBSIF0gL0NvbnRlbnRzIDIwIDAgUiAvTWVkaWFCb3ggWyAwIDAgNjEyIDc5MiBdIC9QYXJlbnQgMTcgMCBSIC9SZXNvdXJjZXMgPDwKL0ZvbnQgMSAwIFIgL1Byb2NTZXQgWyAvUERGIC9UZXh0IC9JbWFnZUIgL0ltYWdlQyAvSW1hZ2VJIF0KPj4gL1JvdGF0ZSAwIAogIC9UcmFucyA8PAoKPj4gL1R5cGUgL1BhZ2UKPj4KZW5kb2JqCjEzIDAgb2JqCjw8Ci9Db250ZW50cyAyMSAwIFIgL01lZGlhQm94IFsgMCAwIDYxMiA3OTIgXSAvUGFyZW50IDE3IDAgUiAvUmVzb3VyY2VzIDw8Ci9Gb250IDEgMCBSIC9Qcm9jU2V0IFsgL1BERiAvVGV4dCAvSW1hZ2VCIC9JbWFnZUMgL0ltYWdlSSBdCj4+IC9Sb3RhdGUgMCAvVHJhbnMgPDwKCj4+IAogIC9UeXBlIC9QYWdlCj4+CmVuZG9iagoxNCAwIG9iago8PAovQ29udGVudHMgMjIgMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXQo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKMTUgMCBvYmoKPDwKL1BhZ2VNb2RlIC9Vc2VOb25lIC9QYWdlcyAxNyAwIFIgL1R5cGUgL0NhdGFsb2cKPj4KZW5kb2JqCjE2IDAgb2JqCjw8Ci9BdXRob3IgKExpdGVMTE0gT0NSIGZpeHR1cmUgZ2VuZXJhdG9yKSAvQ3JlYXRpb25EYXRlIChEOjIwMDAwMTAxMDAwMDAwKzAwJzAwJykgL0NyZWF0b3IgKGFub255bW91cykgL0tleXdvcmRzIChPQ1IsIGludm9pY2UsIHRhYmxlLCBmaWd1cmUsIGFubm90YXRpb24pIC9Nb2REYXRlIChEOjIwMDAwMTAxMDAwMDAwKzAwJzAwJykgL1Byb2R1Y2VyIChSZXBvcnRMYWIgUERGIExpYnJhcnkgLSBcKG9wZW5zb3VyY2VcKSkgCiAgL1N1YmplY3QgKFNlbWFudGljIE9DUiBjb3ZlcmFnZSBmb3IgdGFibGVzLCBmaWd1cmVzLCBhbm5vdGF0aW9ucywgYW5kIG1ldGFkYXRhKSAvVGl0bGUgKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgL1RyYXBwZWQgL0ZhbHNlCj4+CmVuZG9iagoxNyAwIG9iago8PAovQ291bnQgNSAvS2lkcyBbIDQgMCBSIDggMCBSIDEyIDAgUiAxMyAwIFIgMTQgMCBSIF0gL1R5cGUgL1BhZ2VzCj4+CmVuZG9iagoxOCAwIG9iago8PAovTGVuZ3RoIDIyODAKPj4Kc3RyZWFtCjEgMCAwIDEgMCAwIGNtICBCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVAowIDAgMCByZwpCVCAvRjEgMTEgVGYgMTMuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc3MCBUbSAoUXVhcnRlcmx5IE9wZXJhdGlvbnMgUmVwb3J0KSBUaiBUKiBFVApCVCAvRjIgMTYgVGYgMTkuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc0NSBUbSAoSW52b2ljZSBTdW1tYXJ5IGFuZCBMaW5lIEl0ZW1zKSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgMSBvZiA1KSBUaiBUKiBFVApuIDQ1IDYyNSBtIDQ1IDczMCBsIFMKbiAyNDUgNjI1IG0gMjQ1IDczMCBsIFMKbiA0MDUgNjI1IG0gNDA1IDczMCBsIFMKbiA1NjUgNjI1IG0gNTY1IDczMCBsIFMKbiA0NSA3MzAgbSA1NjUgNzMwIGwgUwpuIDQ1IDY5NSBtIDU2NSA2OTUgbCBTCm4gNDUgNjYwIG0gNTY1IDY2MCBsIFMKbiA0NSA2MjUgbSA1NjUgNjI1IGwgUwpCVCAxIDAgMCAxIDU1IDcwNyBUbSAoSXRlbSkgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNzA3IFRtIChRdWFudGl0eSkgVGogVCogRVQKQlQgMSAwIDAgMSA0MTUgNzA3IFRtIChBbW91bnQpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNTUgNjcyIFRtIChEb2N1bWVudCBhbmFseXNpcykgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNjcyIFRtICgyKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA2NzIgVG0gKDEyMC4wMCkgVGogVCogRVQKQlQgMSAwIDAgMSA1NSA2MzcgVG0gKE9DUiB2ZXJpZmljYXRpb24pIFRqIFQqIEVUCkJUIDEgMCAwIDEgMjU1IDYzNyBUbSAoMSkgVGogVCogRVQKQlQgMSAwIDAgMSA0MTUgNjM3IFRtICg4MC4wMCkgVGogVCogRVQKbiA0NSA0OTUgbSA0NSA2MDAgbCBTCm4gMjQ1IDQ5NSBtIDI0NSA2MDAgbCBTCm4gNDA1IDQ5NSBtIDQwNSA2MDAgbCBTCm4gNTY1IDQ5NSBtIDU2NSA2MDAgbCBTCm4gNDUgNjAwIG0gNTY1IDYwMCBsIFMKbiA0NSA1NjUgbSA1NjUgNTY1IGwgUwpuIDQ1IDUzMCBtIDU2NSA1MzAgbCBTCm4gNDUgNDk1IG0gNTY1IDQ5NSBsIFMKQlQgMSAwIDAgMSA1NSA1NzcgVG0gKEl0ZW0gY29udGludWVkKSBUaiBUKiBFVApCVCAxIDAgMCAxIDI1NSA1NzcgVG0gKFF1YW50aXR5KSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA1NzcgVG0gKEFtb3VudCkgVGogVCogRVQKQlQgMSAwIDAgMSA1NSA1NDIgVG0gKEZpeHR1cmUgdmFsaWRhdGlvbikgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNTQyIFRtICgzKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA1NDIgVG0gKDQ1LjAwKSBUaiBUKiBFVApCVCAxIDAgMCAxIDU1IDUwNyBUbSAoUHJvdmlkZXIgcmV2aWV3KSBUaiBUKiBFVApCVCAxIDAgMCAxIDI1NSA1MDcgVG0gKDEpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDE1IDUwNyBUbSAoMjUuMDApIFRqIFQqIEVUCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiAxLjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiAxLjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiAxLjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiAxLjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiAxLjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiAxLjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiAxLjc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiAxLjg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKMTkgMCBvYmoKPDwKL0xlbmd0aCAxNzIzCj4+CnN0cmVhbQoxIDAgMCAxIDAgMCBjbSAgQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKMCAwIDAgcmcKQlQgL0YxIDExIFRmIDEzLjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NzAgVG0gKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgVGogVCogRVQKQlQgL0YyIDE2IFRmIDE5LjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NDUgVG0gKFJldmVudWUgQ2hhcnQgYW5kIEZvcm11bGEgUmV2aWV3KSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgMiBvZiA1KSBUaiBUKiBFVAouMiAuNDUwOTggLjg1MDk4IHJnCm4gNzAgNjEwIDY1IDcwIHJlIGYqCm4gMTcwIDYxMCA2NSAxMTUgcmUgZioKbiAyNzAgNjEwIDY1IDkwIHJlIGYqCm4gMzcwIDYxMCA2NSAxMzAgcmUgZioKMCAwIDAgcmcKQlQgMSAwIDAgMSA5MCA1OTAgVG0gKFExKSBUaiBUKiBFVApCVCAxIDAgMCAxIDE5MCA1OTAgVG0gKFEyKSBUaiBUKiBFVApCVCAxIDAgMCAxIDI5MCA1OTAgVG0gKFEzKSBUaiBUKiBFVApCVCAxIDAgMCAxIDM5MCA1OTAgVG0gKFE0KSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDU1MCBUbSAoRm9ybXVsYTogZ3Jvc3MgbWFyZ2luID0gXChyZXZlbnVlIC0gY29zdFwpIC8gcmV2ZW51ZSkgVGogVCogRVQKcQo3MCAwIDAgNzAgNDU1IDY1NSBjbQovRm9ybVhvYi42YmU0MGVlYWFmMGQ4MWQxODc1MGRjMGE3ZjlkYzIxMyBEbwpRCnEKNzAgMCAwIDYyIDQ1NSA1NjUgY20KL0Zvcm1Yb2IuZTVjMzE0N2QxOGY5ODY3YTRmMTA4ZDNhNTVhMjQ4YjQgRG8KUQpxCjQ1IDAgMCA0NSA0NTUgNTAwIGNtCi9Gb3JtWG9iLjJhODhhYzY3MTJjOWNhNzI3ZTRhZDhmMjViY2JkYWUyIERvClEKQlQgL0YxIDEwIFRmIDEyIFRMIEVUCkJUIDEgMCAwIDEgNDUgNDcyIFRtIChTZWN0aW9uIDIuMTogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNDQ0IFRtIChTZWN0aW9uIDIuMjogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNDE2IFRtIChTZWN0aW9uIDIuMzogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzg4IFRtIChTZWN0aW9uIDIuNDogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzYwIFRtIChTZWN0aW9uIDIuNTogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzMyIFRtIChTZWN0aW9uIDIuNjogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzA0IFRtIChTZWN0aW9uIDIuNzogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMjc2IFRtIChTZWN0aW9uIDIuODogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCiAKZW5kc3RyZWFtCmVuZG9iagoyMCAwIG9iago8PAovTGVuZ3RoIDI0NjYKPj4Kc3RyZWFtCjEgMCAwIDEgMCAwIGNtICBCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVAowIDAgMCByZwpCVCAvRjEgMTEgVGYgMTMuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc3MCBUbSAoUXVhcnRlcmx5IE9wZXJhdGlvbnMgUmVwb3J0KSBUaiBUKiBFVApCVCAvRjIgMTYgVGYgMTkuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc0NSBUbSAoS2V5IFZhbHVlcywgTGluaywgSGlnaGxpZ2h0LCBhbmQgQ29tbWVudCkgVGogVCogRVQKQlQgL0YxIDkgVGYgMTAuOCBUTCBFVApCVCAxIDAgMCAxIDQ1IDMwIFRtIChDb25maWRlbnRpYWwgfCBQYWdlIDMgb2YgNSkgVGogVCogRVQKQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKQlQgMSAwIDAgMSA0NSA3MDAgVG0gKEludm9pY2UgTnVtYmVyOiBJTlYtMjA0OCkgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA2NzUgVG0gKFB1cmNoYXNlIE9yZGVyOiBQTy00MDk2KSBUaiBUKiBFVAouOTQ5MDIgLjkwMTk2MSAuMzUyOTQxIHJnCm4gNDAgNTU1IDUwMCAyNCByZSBmKgowIDAgMCByZwpCVCAxIDAgMCAxIDQ1IDU2MCBUbSAoSGlnaGxpZ2h0ZWQgdG90YWwgcmVxdWlyaW5nIHJldmlldykgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA1MzAgVG0gKFJldmlld2VyIGNvbW1lbnQ6IHZlcmlmeSB0aGUgaGlnaGxpZ2h0ZWQgdG90YWwgYmVmb3JlIGFwcHJvdmFsKSBUaiBUKiBFVAoxIDAgMCByZwpCVCAxIDAgMCAxIDQ1IDQ5NSBUbSAoUmV2aXNlZCB0b3RhbDogMjQ1LjAwKSBUaiBUKiBFVApuIDQ1IDUwMSBtIDE1MCA1MDEgbCBTCjAgMCAwIHJnCkJUIDEgMCAwIDEgNDUgNTc1IFRtIChodHRwczovL2V4YW1wbGUuY29tL2ludm9pY2VzL0lOVi0yMDQ4KSBUaiBUKiBFVApxCjEgMCAwIDEgOTAgMTMwIGNtCm4gMTggMCAyLjQgNzAgcmUgZioKbiAyMS42IDAgMS4yIDcwIHJlIGYqCm4gMjUuMiAwIDMuNiA3MCByZSBmKgpuIDMxLjIgMCAzLjYgNzAgcmUgZioKbiAzOC40IDAgMi40IDcwIHJlIGYqCm4gNDIgMCAxLjIgNzAgcmUgZioKbiA0NC40IDAgMi40IDcwIHJlIGYqCm4gNDkuMiAwIDIuNCA3MCByZSBmKgpuIDUyLjggMCAyLjQgNzAgcmUgZioKbiA1Ny42IDAgMy42IDcwIHJlIGYqCm4gNjIuNCAwIDIuNCA3MCByZSBmKgpuIDY2IDAgMy42IDcwIHJlIGYqCm4gNzAuOCAwIDIuNCA3MCByZSBmKgpuIDc2LjggMCAxLjIgNzAgcmUgZioKbiA4MS42IDAgMS4yIDcwIHJlIGYqCm4gODQgMCAzLjYgNzAgcmUgZioKbiA4OC44IDAgMi40IDcwIHJlIGYqCm4gOTIuNCAwIDMuNiA3MCByZSBmKgpuIDk3LjIgMCAxLjIgNzAgcmUgZioKbiA5OS42IDAgMy42IDcwIHJlIGYqCm4gMTA0LjQgMCAyLjQgNzAgcmUgZioKbiAxMTAuNCAwIDEuMiA3MCByZSBmKgpuIDExMi44IDAgNC44IDcwIHJlIGYqCm4gMTE4LjggMCAzLjYgNzAgcmUgZioKbiAxMjMuNiAwIDMuNiA3MCByZSBmKgpuIDEyOC40IDAgMi40IDcwIHJlIGYqCm4gMTMyIDAgMy42IDcwIHJlIGYqCm4gMTM2LjggMCAyLjQgNzAgcmUgZioKbiAxNDEuNiAwIDEuMiA3MCByZSBmKgpuIDE0NS4yIDAgMy42IDcwIHJlIGYqCm4gMTUwIDAgMi40IDcwIHJlIGYqCm4gMTU2IDAgMy42IDcwIHJlIGYqCm4gMTYwLjggMCAxLjIgNzAgcmUgZioKbiAxNjMuMiAwIDIuNCA3MCByZSBmKgpRCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiAzLjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiAzLjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiAzLjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiAzLjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiAzLjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiAzLjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiAzLjc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiAzLjg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKMjEgMCBvYmoKPDwKL0xlbmd0aCAxNDk0Cj4+CnN0cmVhbQoxIDAgMCAxIDAgMCBjbSAgQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKMCAwIDAgcmcKQlQgL0YxIDExIFRmIDEzLjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NzAgVG0gKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgVGogVCogRVQKQlQgL0YyIDE2IFRmIDE5LjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NDUgVG0gKEFwcHJvdmFsIFNpZ25hdHVyZSBhbmQgV2F0ZXJtYXJrKSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgNCBvZiA1KSBUaiBUKiBFVApxCi44Mjc0NTEgLjgyNzQ1MSAuODI3NDUxIHJnCkJUIC9GMiA1NCBUZiA2NC44IFRMIEVUCi45MDYzMDggLjQyMjYxOCAtMC40MjI2MTggLjkwNjMwOCAxMTAgMzkwIGNtCkJUIDEgMCAwIDEgMCAwIFRtIChEUkFGVCkgVGogVCogRVQKUQowIDAgMCByZwpCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVApCVCAxIDAgMCAxIDQ1IDYzNSBUbSAoQXBwcm92ZWQgYnk6IEpvcmRhbiBMZWUpIFRqIFQqIEVUCm4gNDUgNjEwIG0gMzEwIDYxMCBsIFMKbiA1NSA1OTUgbSA3NSA2MjUgMTEyIDYwMiAxNTUgNjAwIGMgUwpCVCAxIDAgMCAxIDQ1IDU4MCBUbSAoU2lnbmF0dXJlKSBUaiBUKiBFVApCVCAvRjEgMTAgVGYgMTIgVEwgRVQKQlQgMSAwIDAgMSA0NSA0NzIgVG0gKFNlY3Rpb24gNC4xOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA0NDQgVG0gKFNlY3Rpb24gNC4yOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA0MTYgVG0gKFNlY3Rpb24gNC4zOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzODggVG0gKFNlY3Rpb24gNC40OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzNjAgVG0gKFNlY3Rpb24gNC41OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzMzIgVG0gKFNlY3Rpb24gNC42OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzMDQgVG0gKFNlY3Rpb24gNC43OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAyNzYgVG0gKFNlY3Rpb24gNC44OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKIAplbmRzdHJlYW0KZW5kb2JqCjIyIDAgb2JqCjw8Ci9MZW5ndGggMTMyNwo+PgpzdHJlYW0KMSAwIDAgMSAwIDAgY20gIEJUIC9GMSAxMiBUZiAxNC40IFRMIEVUCjAgMCAwIHJnCkJUIC9GMSAxMSBUZiAxMy4yIFRMIEVUCkJUIDEgMCAwIDEgNDUgNzcwIFRtIChRdWFydGVybHkgT3BlcmF0aW9ucyBSZXBvcnQpIFRqIFQqIEVUCkJUIC9GMiAxNiBUZiAxOS4yIFRMIEVUCkJUIDEgMCAwIDEgNDUgNzQ1IFRtIChBcHBlbmRpeCB3aXRoIFNlY3Rpb24gQm91bmRhcmllcykgVGogVCogRVQKQlQgL0YxIDkgVGYgMTAuOCBUTCBFVApCVCAxIDAgMCAxIDQ1IDMwIFRtIChDb25maWRlbnRpYWwgfCBQYWdlIDUgb2YgNSkgVGogVCogRVQKQlQgL0YyIDE0IFRmIDE2LjggVEwgRVQKQlQgMSAwIDAgMSA0NSA3MDAgVG0gKDEuIFNjb3BlKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDY1MCBUbSAoMi4gRmluZGluZ3MpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNjAwIFRtICgzLiBSZWNvbW1lbmRhdGlvbnMpIFRqIFQqIEVUCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiA1LjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiA1LjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiA1LjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiA1LjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiA1LjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiA1LjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiA1Ljc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiA1Ljg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDIzCjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDA2MSAwMDAwMCBuIAowMDAwMDAwMTAyIDAwMDAwIG4gCjAwMDAwMDAyMDkgMDAwMDAgbiAKMDAwMDAwMDMyMSAwMDAwMCBuIAowMDAwMDAwNTE2IDAwMDAwIG4gCjAwMDAwMDYzMjIgMDAwMDAgbiAKMDAwMDAxMjgyNCAwMDAwMCBuIAowMDAwMDE0MTQyIDAwMDAwIG4gCjAwMDAwMTQ0OTYgMDAwMDAgbiAKMDAwMDAxNDY2NCAwMDAwMCBuIAowMDAwMDE0ODUwIDAwMDAwIG4gCjAwMDAwMTUwMjggMDAwMDAgbiAKMDAwMDAxNTI1NiAwMDAwMCBuIAowMDAwMDE1NDUyIDAwMDAwIG4gCjAwMDAwMTU2NDggMDAwMDAgbiAKMDAwMDAxNTcxOCAwMDAwMCBuIAowMDAwMDE2MTE1IDAwMDAwIG4gCjAwMDAwMTYyMDIgMDAwMDAgbiAKMDAwMDAxODUzNCAwMDAwMCBuIAowMDAwMDIwMzA5IDAwMDAwIG4gCjAwMDAwMjI4MjcgMDAwMDAgbiAKMDAwMDAyNDM3MyAwMDAwMCBuIAp0cmFpbGVyCjw8Ci9JRCAKWzwyZjUwODkzYTFlYWZmMTExOWMwNDcwZmM0YzU3ZTI0Nz48MmY1MDg5M2ExZWFmZjExMTljMDQ3MGZjNGM1N2UyNDc+XQolIFJlcG9ydExhYiBnZW5lcmF0ZWQgUERGIGRvY3VtZW50IC0tIGRpZ2VzdCAob3BlbnNvdXJjZSkKCi9JbmZvIDE2IDAgUgovUm9vdCAxNSAwIFIKL1NpemUgMjMKPj4Kc3RhcnR4cmVmCjI1NzUyCiUlRU9GCg=="}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"object":"error","message":"Invalid model: invalid-ocr-model-for-parity","type":"invalid_model","param":null,"code":"1500","raw_status_code":400}' + headers: + CF-RAY: + - a34832c9ae37ad44-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 23:54:08 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05f64-fa6f-76c9-8b39-fd7d42840ba2 + x-envoy-upstream-service-time: + - '2' + x-kong-proxy-latency: + - '16' + x-kong-request-id: + - 01a05f64-fa6f-76c9-8b39-fd7d42840ba2 + x-kong-upstream-latency: + - '10' + status: + code: 400 + message: '' +recorded_at: '2026-09-01T23:54:08.641470+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKJZOMi54gUmVwb3J0TGFiIEdlbmVyYXRlZCBQREYgZG9jdW1lbnQgKG9wZW5zb3VyY2UpCjEgMCBvYmoKPDwKL0YxIDIgMCBSIC9GMiAzIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PAovQmFzZUZvbnQgL0hlbHZldGljYSAvRW5jb2RpbmcgL1dpbkFuc2lFbmNvZGluZyAvTmFtZSAvRjEgL1N1YnR5cGUgL1R5cGUxIC9UeXBlIC9Gb250Cj4+CmVuZG9iagozIDAgb2JqCjw8Ci9CYXNlRm9udCAvSGVsdmV0aWNhLUJvbGQgL0VuY29kaW5nIC9XaW5BbnNpRW5jb2RpbmcgL05hbWUgL0YyIC9TdWJ0eXBlIC9UeXBlMSAvVHlwZSAvRm9udAo+PgplbmRvYmoKNCAwIG9iago8PAovQ29udGVudHMgMTggMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXQo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKNSAwIG9iago8PAovQml0c1BlckNvbXBvbmVudCA4IC9Db2xvclNwYWNlIC9EZXZpY2VSR0IgL0ZpbHRlciBbIC9BU0NJSTg1RGVjb2RlIC9GbGF0ZURlY29kZSBdIC9IZWlnaHQgMzIwIC9MZW5ndGggNTYxNSAvU3VidHlwZSAvSW1hZ2UgCiAgL1R5cGUgL1hPYmplY3QgL1dpZHRoIDMyMAo+PgpzdHJlYW0KR2IiL2w2I3BIcSVSZE1bNDNaKzdadVpbLUFpNUtMMzIvY0tJbkxtaixYPmxxXEVxWFItTGs6PWw+I1lObGclcFhybTBwMGRhSFMvXHBpPC5cPyFyU1Y1PGpoNFMhcj5xc1hSVXAtN18zVjItWWksZChNMkNrWk5yLz9aTUFIV0FwJUthNz8nWS4zRzs6WW4zZlZ0TVtFXERkPkY3b2FlZjgrbkw6XERhWFhyMjpMN25AWUAzJ2ZWWktaMTh0aDtXMiVJTV5yWylmQk08RyhULVx0VCNERidTRWktKCM4YTApUiVEODMyazdNQGksOjU1aT5GZTouckk9amVcNE9rW19gLG0oMEI6MW1ZNXE4RURtVWRdazs/KCVIbS1zcFAsMDhuQzlbWy5yST1qZ1U2ZTIsX2JeUz1eNXNhXEwzXGtqQ1RmZkMhWWRyKXBVVj1iSk9janByOG9FJ2ZWWTBETy1KKlg3MTZwTV5yWykyb0kuMDJjL0pGSDU5VDtFLExuNzlBRVk1J2c3SSpERSdUUVhqSSpvI0pUSHNtb2BuOF9tcFlHVS5ddF1sQ2toOGNmMGckYzsiLWBHa0ZpUSppIjUyaWVlXSMlQD5baXJCMzg6bzpFbzkyazY/XFk5IUNiRWhwdURNXnJbKTJiQD07bzY6KD5EMG0lImdpLi8mXVFXWjFoWFBwYy8/Wk1BZ2YvUE5fVl4+PzJYRlJPRVBvPz9ubGgtaVglMlIuWmFmIkxFcmtpOFNBLl0lRFVsKSdpZURNL25saSEscmFZXWU9XjVzYVxNLzpcZzBhJSVTK0o4N1xEXlxnXVUlS1otVU5UW2A+KD4lJF5UQ2JyJih0I0QiZklKP25QYmw5Q3BiXyRDQ0I5WS4zSEcmcFooWD9cOlFOMzVcWT1RKzI/TTM5U3Q/PjpJY1M9NWo6KW09WVQ0cC8uMmxZL0NNMW8lRiJvL0ViJlY+JDteb1dpO2omWCM1PjxjOC9tPlhjXVs9LGExPkRpOkRnZk8nRFYtcWA/UXI9O1pQXT5fcmpIW1pdNWxyJTBJLm42Q2VYaS0oPTlNWiFFJnJRQWdFayI4a1lmQ1Iyb2M+LlMlUTcrazlZL0NLX1NLQ3NibzMvLS5HX1A+KlErLWdzO2RNZT80bHU9Tj9aPzxsNG5IZV5ZSHN0M3BGR1o+WSZjQT9yVWd1V209KiJwYms5O1hyKj8+J2FvLilZUihHMjdub3BrOWdBO1YvMFIlWExqbVJBOC0yL3FYQC4pIztiMjopPE9uVUJvQC4pbjxha3IwdSFWXCw0QC4qYTxha3NsaG9Cbk9wWUlNIXFQQmlBLGlLXms6PWtyL3BfYiQnLDhdPFVQPWtxPFhvLDNiLFdWKVNPLzhlWzBhRTVPMjVmbl5DTXFhR0JxY3FmI3JeS245Yj9UbDNpIiNPbWNMMTNdViFvbnM4bURAP2BvZWdwUSt1TGlwUmBzVSFTZ2UmJGhOKGhyWjdCX0knMUJrRTVVY2hiXEoka0haQjpcaz1TdFoyYWNIPV1RVTNmZ0RBN1xZL0NLX1BOND0+Q1kvRXBHOSZqMDhtRD9UOllydCRHLWVxJlFkVGlVYCEoKllqWG5cbFFpIkNAMjtSQ2U+UT9dZWhTUXBLXlY8RFhlN3UvKSJgT001KD9XIjpycWBndGNXbDs3aSp0YiFiXEwvVEZeQSs0XElvXm9LdW1QZGYqOyc6XFp1XWQscWFfK1BkYD9NYzE5SnRJU1ddNjRCJDc5bmVVQXFFVzAuYm10cClzVy5eKHFba1AhW29baXFmOkZpLC8tLz5NSVFiVkxrUSw3PllRI0krTD5sSkAlNHBrNTY8NVBjTmxaLktRMCdDOFg+JCFCNW8mMDMyRWpdUUtsMmZCcT8wKjwsPyskXycmJSY/Y0JTUjJDIz4jbEwjTER1UEY5cloqL0taP1dqO3BQM1lOUVRJU20rQlhnNmJTaWNVUidjUV8mbTsyVT1MVHItZDpfbmRhXiZoYV1McD9OOW9rUihTV21ITjNwTzlwMF9ia048Xy5hKUxfQ28sQk1YaEJkWFIncVhKYCQhNUhmbidORCs9OGUrVkdgNS09Z3NtcGU0bXJURCxrJEsmK1VaUWJXLl9kJj9XWkZINl4tU1pvW2k3KUljL3RrU3BRVTc0Ri10dGY3czlScmk8VFQpai5sdEgvSkM0Jz5HWFA/RVg7XGo9YTxjcFxybVZuIypkXl02IkdPUVJENWxJcjNkMl9xckk/bTlma2BqUnI6YEs+KnVWM3BvSy9uczpgLWRwSiM2cGM9SS5cOWpkaT9fNG9aSWs9WFldMCVTJURSVjNJSDFFVGBDTjBcZ2pBPzZYVWhCTUhQQllQYSRXP2E4TyVRcDslRWkrTV1HcGlSS1lnZjg8NUxmZTxyVltOaD1LNllhL20kRy0vPVRTRDwqP0doSkljMEN0UDInPDVLKixwW05CUGsiUzouaj88JWIlUyFWLTdqWnE9R29wQ25AR1lmVzhfdWdhRDBQTCdIO3FgalBrWF51XSZhKmc4Yi9KTUNLb2lOZSJbT25YTHNUSWZJQU5ZOXAuSFI/WCJWaEwpdVYhX3JaO29icyREXUJUZyMqcm4/P1tHNFZZY1JqJTdcYjtSSiM3JEhbQG9bZCtKWkJbcCtnYSUuZT9iJk8xMFVhOCpYTTxtPWI8P3U0RTpgLj1JLj9kP0dWaSJLRS4/K2tTNHUqcWlNT2NwcSJkKjlTLEE5QG9vXkNlWyJDanBnaF82WGJOSi1DakRXQ2BHXWtSXGRrYUpNYTYlWW4yWWU8ZCxtdGNKUjxzXD1jSTksQWhwQ2dQSmRsU2dTcmwxVz9fNHBgcmBWKzM5NnBlJCtGZVwtIXJUSCQkS0hLUUI6JEE0bGdfT2UnRUdMZlEpI2JrPHBQK0tZKyYvRVdrQEFNZVR0TkJsZk0nUzBeN0xyckpUYF5pPSM1U3FtdT1ES0Q7JFIxSHRaKTkhJT1SJUBTT3FoSHFjUkRhO1hWbmllK0ZCOl9pczY5WWpdKVkzNmRiTjQuNyI0Jyohci5XQUksOFk8WGU9SmxPX1ZEP21qaUxgakRpKT5WNGFaN1ReRyVLOWhIR0YhJDdHTjVQWlJrJGNtM25bckFFcHRAbitlOW8/MUpwUk51ZVZJZ3I+cUdnRlshYj9scmwsXT0/RyVBZSNbY2ksYEQ/U0lvb0RdYUg8VTJYTWQ5O2UmaT9vMUkzPHJROjpfR05KIlNsPlc/ZDNPIkxxLm4oWkxcLzYhNk1WIVlQTjZURVlAVDIyaFhmVVdqPV8/RVAuLkAoUURYLltjZUdAXF5ML2xpaFI8RGVObCEvLFQ6QUZOSyRiaWg0OFdlIiQ/LFpkR1M5Zls3WyluTW1dVCYwXVt1Y04pPDxGZjlwMyEjXkQ8aj03YUU2InEkVFpVakxyRTtNYzlCZG1gQ2QwNEYobDhmLixWXkNaImxxXCw+QDBIXWMndEkxP15OKE9YIWdRUkQ2LXJZaydDaSlrOUBwRkNgPW5tOkxfam85UFdFLV06by50O2o4aFdWLlNyXVIxZ0gsOTUqSXNMVGtsRUI9STRsZGw2Xz1vSTBIMUtoWFk9OEpANV5EcFBtb1RNWmpMczhTTXUzTm47cm4yP1lQZE0oSGI3JC5uVHVzcWpUWFBGcjhsJF5eZ0IyIzRdREUsXz1vSTBIMUs4TWY2MldTLFh1LDByUThLPV1PVWxJcCMhWHFQOEFWXW5zOmBtLF8nW21IT3U7MEsndWU/byZMaFJZUEFrOmI2dSRrcTkrdEdUImpEYShiQD84SWNXNFloTkBma0pcbWM5SUBtJ0NTa10zRW5DUClfXixrX1BpJnBVSW5vb3UrW10mXEddUlFkP1JISVguYWExVmYxITsiYW9lQF1DbHVaRHFmKGVSLlA0c21tNkZDLk9tLUU3Im4kJG9aN0luK1xWPDsyS3MnRDtrJlBcRjRZZU5qJUFkPVs7P28xSTNXVyRlVlMhbVheSyRiaWg4LEJoTERkSWhlJFYkP0BTYGZIMzRXYm5dMl4hVWRCVGckR3I6Uy8uXXRhdXU/U0lvb21qKkJHOSJvPi4kSGxkMVJPJT49SiI4QG9pNS0xb15ML2xpaEFRPWgyPmtfPC1uS2thQlRnIjFyKjlzKU9VRnVmcmRyTkNII05HJDxQPy9XZmE+bzAwVWE5VWVjJ0dXcFAsMDtZbCJbIjhiMWEobiIrRmNcVG5QKj9TSW9vbiJwIyRNdFc6JlVwMGIjVUlrJTVrak9AOypjP19MbmVbNjNhYTlTbkxONURjaTxtYjgwVWE5VT8yZ0YyamxhVzUvOllLTFAuVlYsKV48Y0UyYypyWSRWJD9AXXVBa1dRIz5RUEQjKTg8cEJsQ2NoWnJdP3FoSkRUbyZUMyxEaG1OJm8vIkhwMFVhOCpGKzddXTRtRGUzLGs7YjEtSFkjcWhmKmlPSFRqSE5xdXJpZW8hXDNNNmYuRitbbmU8Il8uXSJFNG9KKjwybCxXTVpjUVEuUk51Zj1KK01gXGw/VkVDbmVbNjNha1AodDwtXFJtPzlQNGEiZiJaW1tkbEBdaFVuRWQ6K0xoMUJUZyNWcXV0LlxoUiE3K25lWzYzYWtPYUBONF4jazQmN0pZXy5dIkUqVzFXNVMoK2UtcjoxWV8tSFRLWzAyZ1o+U2owKjZeTC9saWhIR0VGbnR1LUk8I0V1YXI4bCUpa1YkOSFtJS0/SSIxamM0NEYrIiovV2dqckYmZkM7QkFmMWpQSj9vTDxdSy9qVCQlazdeSD9mM3JtbSlnVldaWmxucz0pUz84bWZINlxBSEtkX1cmI3JFY19vaylvQ2I8VFA2LlhPY15TPlc4LmJobEEiWzhbbi4pYW4qQGNqUz86WUgqRl9pSS5nJ1tAPG5vInJkY0U8JGgxV2RdTiUrSSlqJWZzSCpkSGQtI1s1PGE8J0NxOllvJyM0XHBSTzRpOyR0bC5QLTljbGRWcFY8U3FxWWwqVilTKWdiTz9FMFQqJEhsZUpdR15dcDdeVXNHNj5qNktuczpfQlxBLT5KbSwxX2EyY1g7XmJFRm05P2lPNGRuWWhQPVZtLSpcYVcnSTRkUW1OclcwIUxFWWwqVikpcy1QUi4oOF1GT0lNckpyOGwkPmlwWGt1RzRWWW8pbTZGQ2ppTWw7bikhZ3NvVmRpalgwRE5gYWE8Nz9wPkdXOVcvcXNzWWwqVilIZkZNOWFMMV0+TztrJSNyOGwmNG1JMCM8Rm5WYnEpa3NTN2ppUC10bitpPi5wU2ExI2hRP187YWgtTU1hRXA9KU5zYEZRJUVpK01dVzskJDRlbDxBOydHXDBCREtgN0dLSl9qSCMqJC5fVksqNnJwby5DcVxwciVEO0wyXEhfJj5lVCJJZ0pBKidpO0hQXm4kSDI1WElmQWdlWl1YPCRnajAmKS9Jc0xVNm9KXS5vZzpUQGJRWW8rZ28mUDVmRGolY0BHcVw6VW1tQCRFakYtJyxUIk1JVm4pWlRLNy9sYzxyUUYodEgjKiJYYzFUa01yOGwmdGtcbW9CWzxaXVJEVldqb0gyI1oyR3JQWjc0bXJCUnI7REV1YWgtaixKQG1cSm8oUDs5LGskbzIxXC1SYmIyNS1rUC5aNm5hNWlmMFErcWdCOGIrcUBTV249OD04a0NYOGIrTTRyMmI+MmsqXWosbz0nK1NablwlXFNyTGtMbnRGSyRtZF1vSE1Dc0goXGBNa0lOU2ZCT3EpPFdoLk5dcidXYS5jNj4sU0VLS1g+Tj1vYC9hbzAzcDNKZE9lUSFkJTNsKkdjZW1TMTM3NyJcdUtENHF0WDU2R3FqVGAvXEgqISluVD1yJGNqamRwVEQpT3EmP2gsOklDPzpNWWtPSjstPGZlRDxGRCxBK0dOU1J1Oj9NcmU+N2hrRSVMak1ZZTIrWEtfVWNoanU+S1QuPW5pPmJvNy1wTWAySVgyQTIhJGZmKTxRRzRnJUluYFoiOG1FMkxVVGpbUGhGZVs3QjxLMSxFPC1FaWhuP15MMjZIIjU+U29EKEBKUk8/Ml9Mcl0+X3JqS1ZDRi1IPTgxUW8xbjYoOUBHRXUham1VWFNeNS9BJSxxYi5HVXAwZGltdW84ODQnMUZZXC8+WHM/ZnNVKF5ba11lJGZjc2wvOlRoW11fOksvLHFiLHFYMEROQCddKz5CYC5gMTUncGNWJVdpcWQ/RUZJWVdrSzddM0dKTHNAYEdOZjNxV0JuVG4qYzR1J1FIM1VuLDZNMUhNXWVkRiRhbDlFdDlXbXJlPU0uV2I2PktGJjFwbXJXXFxBWVwuc1ppLDI+XXIhZlZbcWdRTCljZVknUnEwUTNkbSFjbm9jZVhzUnE3Qm9RcmRNZyxjWCEpL3E3QkwtaVZlV3FUIj9oPW5lV0VlQEYqcmlhUyFrKVxcQXUsJFtXWmA7UWInclJKaGldPk0tOWxwTXJGZzhhQSQtaFNQbCddcGVoPDhpY2hSYkY8XmppZjwxXydhRWovM2UtXGpnO0xONyhXUStyJSppV1dyZyhXQDZnJSdIKHE0NnQ1PVMqVCRWUWpKPTVxKixoJVUoZj5UODFePl9yaktbU2xsLlxHVnVaQmtfXXJLVGYoSD1fbU4kUlYvWSk9c1UvcDwwS0V0QC5LMnVfXyVNSm0oREVcSXRxVSYucmYiMF9haCFXVVx0cG5BP05rKW1EOm0wYk5qRjchO0ZccT01cSosPD0xLDhGWyJPPkhQVF08LF04SVUpMHUxUWI+SSohXER0LEVLdTxNLiwqX2dRWS4uJi5ZZW1ZalNWUVtcWmFmIUFFS0xTNWs0S2guOF5IQ1JTJkMiMCowa2E0XmpVTUU9NTUmUjdjQkhBWCUuJFdaYWYhQUU8LUU9a004Ky5cIjpQUjJpNCQrbjZ0MWVqIy0xKz5dR3Q7Vz0jOVloKityZVQjREYnTTQ6RjRHMkFFLVtJJSslRUtkbigkTGQoPjRBPXEwYD4oPWdTIl09QGhYYjU2bW9gbjhQSVRRWVlOPzlTNWxbRnAnZlVzZ1g3MCRELi5lNjtNXnJaWDJjL0lySXJGLGpaYWYhQUVGW2w1QENZV0woMEI5OkRXJTk8IS1FLyppZWo1RE5WWSVfJ2V1QG9FUShaMypyPlEkUmolNnM0YzpoLicpMD5NMnIjWyhuKk83X0VLZG9lWTVRPmFQUD1ANmA+KD1nXTRdbkQyY1xfaEhQVF08LGpwRDJeVWAlZyVkNyYjRUtkbzVwQU5uTmpJT10pXERlaVRRUTFEN0BjU1FZWmFmIUFFVyU2YF9oSDJvV0ZwNGk6I1wzXnI4bzNiKlMuUixpWy9ZLjxvVzUzZjxgIUgtPzI4XU0zaUVzY2YwaE9gOWVvP2U9T1ZvMnNMIT5hWkkiRkVRKForSUg2c1RaYWYhQW0zX0NhbW9gbjhQUC9vcHJ0Iy1rPW0jfj5lbmRzdHJlYW0KZW5kb2JqCjYgMCBvYmoKPDwKL0JpdHNQZXJDb21wb25lbnQgOCAvQ29sb3JTcGFjZSAvRGV2aWNlUkdCIC9GaWx0ZXIgWyAvQVNDSUk4NURlY29kZSAvRmxhdGVEZWNvZGUgXSAvSGVpZ2h0IDMyMCAvTGVuZ3RoIDYzMTEgL1N1YnR5cGUgL0ltYWdlIAogIC9UeXBlIC9YT2JqZWN0IC9XaWR0aCAzNjAKPj4Kc3RyZWFtCkdiIi9sOTYpO2cmT2BoPmVYVDgrZFFNWkZQbm1RMU04ai8kJDM7ODVOYzljbTkjSVk2JWc2MnBGJS5ybT9YTmoycWQyU1I5MXFvTmxnT01pJyNVYStVUiUyXWQnPzQmOkhlYVJEZDdfWi1eNWdYXERlUXFoJiRJbEJcWzY/YEdOTktQPUQnJ10oVDkzRVNHRy5VXzBEX1FAME5NZ2JyRVwvJ3JaZG9dcmluaWpSZkxEIixdbzwoRjhZTV5yWmFZMGQ+OVxTOj9tZTdXZD9EOFdiI2kibj1CKkd0WChZLi8+WTk2XkUkUyJQY19JMjVvPipvKWA2UzNxMmRTJ3AnZy8/WkwiXDlqU0AqQi1xPD04LGomRVNHSEFZUEFFQFMzJCoqWmFmIydpTFg8RGNlS1hjPmhXaDFgR05MVV1bbXElXS9qaXJaYWYjJ2lMXHE/LE0tITtmQ2A5T0VTR0c8bV1yRExXL2VMZC8/WkwiXDZjWGhNPGJQX1U8aGhWZ09uaGhZMHUtLXA0am1eNSlVcS8pbTkocWk3USt0XidGUiJNXnJaYVkuSE5eL1Q3MEoiTUUoP01WRWspK2NVLWQpaSs7Vm1eJ0ZfW2MnW21qIVQwTWoqTV90bUJhPV5bYyw0dGojUWhRaFkmWiMvP1pMIlw9cT5xNE03cig4XGROSi5ySDtEWm5yZ09EbGRna1hqPyFhZ09nSTsoQSZYJGpmQGBwYHAnaD5palJnQzhgT2dgTDQvNS1JMjVvPjJWXnU/NFBucEttTjguS2llbEtGUmswT0heYDFVVzNuVl9aPl1KVDFhaT0rNE5xZDRFM19LNiJZLi8+WS0uNyNeXiUoMV9cIillbldUSGguYG5cSC4ncUguNSpHZmRDRElbU09idD8nOFVDTzw3JyI5QU1zIjNISGE6KFdeUl5qJz8wYzArKU9sVSlALnJjTUwmTURKV2E+cD9tZUV1RWtnckEuP2xnbCNUJTlLJTRwS1dSMFIjPkk1P11kLnJaakElamRhSCE7XGhEcjZYakdNJS9EWk90OStCOnNxUmpXNDhDYlVUbmwmYjw/PypxQVFLOmhLPFNmLCVeJCc3KDFaK0Y/Uk9gQHNGLGNOPGZjMFc7IU02TClpanMzdWY3RF1yOT4vMiY/WkYmLWs6VSRDSCI0NmxqbFwuc1xGLy8mcTNPJUs+Pz5qWmllbSdJaDNgUzhqMDQqJms6PF5GbWJHTERRVCYxJ1F1S2UhTVgvUkRTcCNDc1onZlVwKWsnaTU2TE5lbzRhQTlIWCNQTUM9NTtDZyspY0VMWS4iW2IoMEI5dWpZR0taN1dhRSRxXmVuWzVLZGk8amMvLEYoMEI7W0NATSxsRiw7aWcpTGZZMU07K0VQaipmYSlQckkiQ05eJFYpMldqai5VRzIuWFJsOmhIJW1GKFMyV2hTRFVHMi5YVGY1YCwhLXRgY21FMFIhWW4qI2xcXk1NUEI3VydFaVxvcj04WVwhLkYwLkNwOic+Tz1gLDNeLVB0IzM9a0BTWEtTLiIlVU07K0M6L1dJV1tjZk8iYzRXVUszJyI5U1NnKS41Q1REUT0sRkErWjgtI1I0MU5caCs3NSRiZzA0PWxUKSciOVNTOWNBXzJUMTdcQSojWlV1TTsrRGUuPzIzVzpHNldLQXAoSyYua1YpZGlAUUhRaFBabydUJ0g+dEZhZFk6YklMVXQtWkFicVJRQipoJkpfY29mXTNNOUpsMmooNGVncmVOZ1tObDspbWshXE9UV1NvJyk7cmlpO29xUE9qWzRcT281VTVrWDtsXDxoNlotYFhXJCV1Myo4bEBXdShPL1RRMmRPPUpSNmZYMChvWEs4Kiw5QUpfKW1lNl41PjlwWXEjM0loXDtsN2lOUFtrREFEWykqcXJkZ29vZFwvO0ZOU0FlUE47WkVidVtFTUhVUGgxOHUjKmU0LjdHYDJgRmtbamEiaSQwJEJjLGIuRU5YPmZKdXJadFBkUTZ1MDpgXk5jXFZfbD1iYVhpYVgtSV09P2ZIPkZgP2FgSCxzRFlYakRtT0leTzouKDYrclAxISw3RmB1NGBCPD1WOGIuPm1RNEtidV5Ga3U8Ij5pXiwuQ1hbVTxRSz09NEBjLzxucDVTVjlqVmk+UGU9OFYtSFlUS3FhUVhaaV1cRT1JVEtpRzFlRCJERF8uamkyUXNXOjosdFYoalJyMiM+b2cucGFnbGA0RUpQc2BsZz40VzNhaFItRV00SjNoS1ZKUWpQQj1GbjorXU4jQWReQVk0NGQ9TkBDO0FuIilJbXM4LnM/QC5QIk9tNy9VRDlbUCwyXCVPTm85OkRwSVw0X1VVbzo7VUludCgpOXBmRUhNImUxajldNlxZa2BZOy4xcFdzbEpJVi9FZ2ZYcz8lTkRsJCJWSikuKjAmQEhFZl1RJkhqc0IhU3JLXGpNOVtQLDJddD8pdGFVXUpIYy0qb1lQSyoqRT02LlVvUE42bFlgcGVjQEkyNF9aRHNtN0o8bU1SRURhJ1U4WUhLbT41PG8mTWZZITNTTFhJI2ciKjM2MjFlRCJ0RFtcbCQ+LU46YUVtTFI6NEJePUxha1REQ3BxJkddQ1xYbFFsOU5QSVAzWDxPaU5aa1NQckhqckhOUTIoaSh0Ry1IMkRTZU46SD0wIVY9R1EjaShVdSloLl9HZ0RdZkhXLW5JcURiU1AoSCkmJi1yLltPXF8iS2RfZ2NJMC05W1AsMic+LSd0T3QoVWlwPVssJ0g3aiwwZDNPIVldSjk+VSZwOyJgZnMuRVZHISYwR11FbTpfaFk9XSYwNmw4NzJzQFNMS2YzKWpSJWQxQ0FwUnReRDk7J25tQl1HU1tvX0ZSVS5PbyZIWGxpKVJIZj1pbzZEVz08bzA0PDhiMiJqQSwpVS9sQEE4Q0ZBJTY3Q2pobkw4Yi8wbGo3My0zT1cqbGZVbydeVGlsVzM9MWVIT2pEajcxJlthM10+T2JVTjJMKmsjSGxdJDo3XyJOJldONyplSGRhMDdiYklvVnVgMk5qSWNgOHA/Zy01VzxeJmlSaU1DLUgnaS9KbStMUHNYRVY2W1InaEZMLCJqOT4obiYvOVxNbUJhS2wpQzk7SVVJayU5R1tuZzhWby81XT1LPSQ6MmoyUTZIKGhsKmA7KzlOW2ExOG5rYCEwOk80Ty5HclZfYGhCYDkoIiguKGY9JjZkKFIyKEFtYCJEPDotS0RaRmptJlApKTVWPl8qX1dfMGkmX0lvQUY0bGZEcWcpWy5uJj8lPlYxJTNZNU9FdT4/OUErZzUhLEsxbnEpM3VcOktocV89Zmw7XEBkcl8tSFYyN05TdHFlRzZRYWFUWkhbbjE3QktXKCNCZEYiNXQ6KWpiPEInXUoyaj1qTHJNT19rISVsaEIlZzJuWSsvaVlJbFNZUE45RXUpdDNMLFpwISYqXTsxQTEuclMsLyxeSFRiYGMrVEUqUSVPRzBOViJVSCIyXllhYTlkQUw6YiRGU2cxJmlUIXEiWFVYOj80MEQzPihBIVZORXAzQFM7ViElOzwxN0JLV14ic1VvRDdXclk1KTJiSjRnMkxRNz9jZkdnKUlyc0lHUVlXVDsvN0FTbGFkOVUuT3E0bVokMDg1Tzk4Y2NPJiQ9Y1JXcC1kM051VkdbbjgqR008PmRGK1EsZXFPR0kqYi5FM09ha09dUG4kYCkoRitjYFQ4XSpdcF1KMmo9alIpI2ZpR08tRkgoUW1oaVNKM21AdTtWVGdXVVZTJTFndWRkY1kuVFxHJy5fUiVkMUNwQEg6Q2NgY1YpbWRrTm4mXmtYZVpiLyJdUE42VEZOVCU8ZW1PP2slakdkXCw4YCRXLCkiLCUpKk9YWGAyajJRNkdvTSo/XEc5WlhQXSZoRDg0SmcobUhvbmxybVBhO19KX08sN1BeRFhEZ1FFODkrY1QrJloiQ0JCNmUhRS1IU3A/TkZAcFlNW1VkPXBISGI+O2AlJ1AqY0RJbjZxInFUTDNvUVNOQEM7QTA5bkpORT8qRyVeSD5YO21YOkJbVkopLipDdTRxQnInUXMjKShmMidHZVNxTiFSRmooNWlEQS03WUVnXWRKdURlX2pIaFMuclVZbj5DKzFnKkhsLHNUJ29TISljSlJiPzxWN1VVTVlaQUlLTXBubzkpJjBCVGckUGhLVGpgVCVFc0VTQGczcURiU1AoSDAoI283U0RnPTpJNCp1L3BVcmEtSFJjVl1TbFNHcCQwa3Vqa1ZHNEB1O1ZULnJUZiszWGU0TlohaU9qbzU5ZiowaCU7ZVByQ21eaDVIOCZZKlFeVDc/Y2V0Mmw6bEhWMUcuXFdTJGNaVExuXnMpalxSI2RLJVUvMDZxJmItSFJjVihmUD5saiwxTGhPdUZxKlhmcCw6PSc8NTNdT2Q8c29ldUNXRio9W143NlcyYWBtNydwP2QkME43bigxMnIqJT10TkpsbC5Dcyg8KiVEQ1pvZCRFTzYycGBncWpVRmxERipmJW1bP25OXDc/Y2V0Mmw6aytlLGMlX1dTJGNaVExuXnMpa1kjb1c8NHNGQzw3RG4xZUQjIWokVjcnbjMobW9rMXFHMkB1O1ZULnJXKFlxKVMtNWNfS1VYLnNZV14tSFJjViRvOycyMDc9MFdtcmxlbDtgJSdQSCIwVmdYLlJBUUhXJ0ZTJ01YYVdmYyw5Vi5dMy5eSUtNcG5vOSkmMEJUZyQ4aEplVXNdLCYoXEghWmhVUDtmQFBQLls9QUUwU0FTMCJrMGdnQSlmMGAmcUNQVVJeKWIpdCkuIWY7UEIoZCpEQT4sTi9DPylbUDUyaE08az9yUURtYz00THIvQHU7VlQ5QkQvJzkrQF4tR3A9MiRQKmEzLlJGYDxIKlYpWlRSTnNQQzJ0QVVXTGNYVVZBMkhtN3A+LjNGa0NCMzhHcD0yJFAuLm5aJERkZC8hSTVuLTlwMnVqMnAxUz9fRF5KN0EySG03WTIwL0A3ZXBaV05GWGs7aE5tI2M/SUphOD0vZVJEODRKZyhtSDRxOjFvZ0RCWTc+aCYpLDY6J1I1XilRLi11I1wzb2w5MjFlSFAhaFFVNlZLS2ElSzZmKEAnUE47LGklSyJsXk8/VFNBNmYoQCdQTjZUPSVEK0hpcVJqMVBLS11xQFNBVkpPXTdEPE5IYjI0O0RdX3JOJ0BcMUQsYm1zJT9MPGAzSFxeMjhCUV9COGFfNmpNcE5nPzxcY01MYjwjdE8hMTdCS1dWO0JrPUJHXj9wVj0haEMxN0JLV05TWlkqQF4oOEViYSJBbmhPKCQvbm06TD1fb2k1PEI6IV9mSDAoIzdSTyU+amhKZGBOIThELjViaWg1RG1IcmFvI0dGYFtoVm50XEJkIS4xLC02LHRIIjJeWWFhOF8jQC4tcmg/SDouWDdQXkRYMD45QGJWMUlGYlRdI0IxMTdCSUFcMlBDZSFUKiJRcHNUX28pdTQwTFlMM0M9XStBOFVQTTFgPiNNTEVMPiRZLidKb2InKyguXkMqOkkzMTw4aychRlJRckhYVCcuO29Ncl9FYmF1dDZOWWYmXFNVOysoXVFKMVBdRGROPEQ+aSJRVjFlRCIlRGFbNSNcXi85IUQzO0puTiRjWilqLEYxVStTWFBxOTkjKW5QMGtQIzYraT0pJUhLXStRK15KbzdQXkRYMDdsLSdNdWhldWVNPS1saEZMLCJqPVUqcVZET0QzXU48U01hVyY/OFAuXDNaODxzSSNUPTwiQk5aXSo1VDhkTkldUzU0YFFOaEsmOldmRGwrOnFQRi9TTDxbLS9MPT5Sb0F1P1x0cVwsW1FbTDcsXkshcDsoKW43NStibkNBKiZcIVJOdWdXMmVvMT9naDJ0MkRRZ10tNz9jZldvTD9BSk9TZUJFVGhVaDNWSikuKkMjMTZzIydjYVwubyUsVWBZOy4xMm9jaltkUW02LEhVW1IzaE8oJC9uZzhTYDdpS0hzTltYWUliOFhBOjhgIi0hNVkyJ2djVSRbSUlzR09dLUhSNGklRWhbLkAsPz87N3MybGw7YCUpJitFJipuIUxVPkJmNV9qWGEsNjlkaEMuUSklJSg8PUo5ZWJJPy46JDk5W1AsMmg3WzZlOlk8PT5cYGA3PjJsOjhPXVk8NyxPIVNXIWhWOkpAYTtcJjc4Uj1vKi8uRWtAPzNdaSszb2w4TFJPIyknMmNgRVwyRDVBXDpFQiNHJ01YYkJgPl1hYCNiLjA5XTVfVFZgWTsuMXBXc2w2Y2NgUDxXT1QhQS1aO10rLF5FMXVZcTZGR11QUUIiNmUocCxkM08iSF1sPjUqakxnLk5xYmJLVDtgJSkmNGA7PDQiJCYiXmY2RXRaYFk7LjFHSmJBVGBXND1YYk0+LWtIL2ZXNlAxU00vXCojKCNhcj8rTFAuXE5jQlRROyNIL2YkJVAxUykjNG49TC1HVD9RY0dvYjlYSCtnbDwtRWpxLzkmQTJmUE42bGFJNyRHKmY/SkBuUk8lPzdEWVtEN25FWmZvOi1iJChyUDEhLDhDXTs3YEI8PVY4YjBVVVFCLmQqWG9iNlk5SmwsZCslIk07PEJbY29RKWY8MTw7MEdkbjdwSyklPk5Ic1YkWnQxOD1uIz1cWCNIUzNvcSojXS5BJE8uSVJuPCxdQU1yYjRDTDs9Uz9CbipkUi1kUC5XKmhXNkhpcTksWjotLUVlIlVQLipndEttQ1hbWEsoNFttLjgtVU5QXCYkSCZpNkFCXjFbPFU5IiF0XUE9Yy0vcmkhUk5SMC4+a2hjJmlPQlhuR1s9KkswXVJiJT9tSShXMVFbLGEqZiYiMmYpSi1FS0tcWi5fT3VNLyg3aUZDNWBAXiciTmdbUT0+PldhPWE9RC1PW3JDIVwlRUNmITdXX3NWYklMVXQtWj03W29fPW06J2MyTy1VKDJJZFYyV2k5UD8yIUdEUik9YU5Vc0ZxPW51PFpnUmFuNiVYSSo9Ti8qKkA9NXBbT1ZQdXI+ZldNKDY4cyRScltsbFo/KlYuM1ZYaEtiMlI0RTozRXRkRHBhcjs7LUVAdCgvLSNTJ0lab1s5LWVfMyhzWygtZ1k0Wj89XG8jPVJFKnNDTSxNOytVQFZbZSFpPzcvNm4zRVAycUUkTmpVRF0lZkVxOGRWJ0RMOE91M29nYENTW1RWZVkvRzInb2k2ZFtFUywsPF9zdW5eaCJGdUNGaVlMTElDP1ddTVJhQS80bm5ZNyM5UzIvcHFEWyNBWmlzZ2NaTCtBR3BCSyQjXm1JaWd0K1MvRGc+aWZYKGdhZFA/MiFVbVwpXSZOUmBKKVQ1U2dYRUxGQWxkazVSS1puX0UpYD4oPWZdL1Q/UGBhSnR1YzwiN2pLPURrYTA1anRoRnNyP2U1KVVxb1YsbGJkaEZOX3RAQHRMUmYydFpeSl9WMilkTz1JZ1xEbDVjSU49ZmhPWD8oKTUpVXJaO0FoK2tYWitbZSZUa2JITUFzTEFXMiYsZS0tUC1cL001OEU0N1swZCteLyc4VjQ3JWhlLVtjWnFndUw3Qm9iXU5EYUFIOTZjQmQhSlFtQTc1IipcKWVPbms5NF1nWE9rRV1KSWY+bjdAS1tkXXU5JjAsRktNMXFuMS5nP3JGIVleTjAlPG4kOXB1WGJCQHUnXi0jUyZeSmldWHRRM1ZkSGdPPzxEQWArY0YuVnAhaT8hKjkjJyI6JDVuOiZAPFNWSCxhP1pGJG9pYDwkJywtVSwoTXNZbzInZlVxUVUoNnExRyVuMU9ZSU5RckVWJihQcVc5WydeViVuUE1WRilaWDpFOmBwNzNIIz1eNXM5RUFRLyhRJDZyWytGUCMsZ2gsPXVGaCRMPGgndTBsYD4saigyYUpKSUdySVg8ZW45IUFEOHFROWEnbWk8bS9jb2A+XUg2UTZlL0g0cForbVFaZDdXZ2dQZClMTmtOKVxZP3RdNy8/Wk0tXDNZJSoydEVSLyJNKWs8TVZGKC9Ub1JbOS9wZFRlWmFmIVFqMTs6JC5CYU8rXlgobnNnUGtJISVTKEVDNSlVcS81SGFHc3JGcmUtcUNKZVpjKyNCTWZ0bjJFV0goWShgPixqImZCWDE1cEBpSnJqMk85QC5ySTAyaVAjNislP2VgLUkyNW8+QiYmUlhyZi5wRk5JMU9SWS4yYGRnWGdobyVAK3JgSTI1bz5CJigpQ3JsUFstN1k8bV8uckkwMkwsc2QnJztYJEtjZjBpOk8xVE4uSXJdVitkQUc3TE1WRikqMm8lQDI5Y0JUIS8/Wk0tXDQuIVZlYnIuakhoOkhHRUxVciIyVFY0U15IUlErRERTRmEkZCRoMWZrNTxERDhvIy43YjloXi5ySS1xTlxQa1M9XjVzOUVMM0trVjU6I2JwJkZdYk0jUkpsM1NKSH4+ZW5kc3RyZWFtCmVuZG9iago3IDAgb2JqCjw8Ci9CaXRzUGVyQ29tcG9uZW50IDggL0NvbG9yU3BhY2UgL0RldmljZVJHQiAvRmlsdGVyIFsgL0FTQ0lJODVEZWNvZGUgL0ZsYXRlRGVjb2RlIF0gL0hlaWdodCAxMjAgL0xlbmd0aCAxMTI3IC9TdWJ0eXBlIC9JbWFnZSAKICAvVHlwZSAvWE9iamVjdCAvV2lkdGggMTIwCj4+CnN0cmVhbQpHYiIwUzk5LE4tJi1VQFxFIXAkZ3BWMjVzMGFrMXRuNyYxL2xDZkBjJmMmSVdZbGw5QUpWaz88RT8nSVdzI003Qj9BUzlPSD9qJUZDcWg3OmhvRkArSiVYZ2xMUk9Cay5uPTJNJXNITUo7bFFaVWdRNl4xLEZvb0BIMUJEVVM6dUdyIkIsaFBRS1ojJixIMyVOJGYwKz1ARlNlV2peTi89MSI4Ikw8RTs+KiIvbEdhQ1IkTW5nIW5pSyU2Z0QpV1JTJkBEKSsmXDRSMForQXM8KnMmcDcsY08yYlpeSz1baGFkbTktKVZmKFxhKEQsXzxXSSRXbUFzX1peVz5kam1WSDJlZjZnSGhPP2BpOExnVlRBYDxON1o7STIiLm1QPFNzIVguMiYpNDJbMypjSFwyMVNKZktpQjI+QTBjNlFxdVphajE0Qi0rQTlmaUc7TDFETGVlQ2FgSDJSPm0vR2VsRkJhKi5TSD5RNSZiMz9scSxnRFdyXTBPbD1UTFcuPzkoazhEY08mPF8qWUM6RkpvXE0jZ20sMkpZRDw1RDJrbXJLcj1fOU9kYmwiKEBZO1hLQFMpUHJsVlFNKEQ7XSo8MWYjQHUlXFR1Mi9wQ2wrPScxMy0xaTwwRl5UTEgxLGUkV0A2ayM+aVY0NU1LPkhGN1BGPW84XCNBOyttIVBGLCFVJE9WYCwqaTFRMUhZKCFpYGB1LjJxXVlpYVM6MjlVNGdZIzkmXG5YPV1XM3RmLEIqNF5gclNQbihDPjEpQVNbSHJPalIlbyZJTjNNbXFsKG1aREooPylFN2pSRVRwPzUkU14jWmFAQ05RUEgsJVtQSXI9NUg2QyckOkIqZSZRZTk0OyloY2pyMSdzZD4hbTxtcUYjO2Y3cUIsZEU4cjlcO2MxTC4+WUQ5MUZIKE8sUXFYNyEzdFpRRT1GXXJYIXM9XC02JzFEWCteXW9GbE49XGNZZDFDbVc4czF1dXNBNiRjT2s4N19pRmcqWjM0Y1hmdUYzYSplPzo+Jkw7TTJJRDlOUyxgYmwvVGQtXlFFS21NcGFeVGhoaDQoaWpyIW9fNW9NZEU9UEBXdTxrNTM/PitCMjtDM19XJFhDUTM/PiJLMjorQFNWKHNwTj1dTGpjczNdbDIjTkJ0JCdQRDo1O1UuI0hyY25yIVgjVURdNDZPJCUvKlAzU3A8NzspPG4hSS9ENF0/WFdON246Zyw/LkA5UCVpTFs6KkJrTlQvNGxDQC9acSFxaSxDZT5kZGtkNExsbmUxJl9RTV9mPjtQbDdWRTFNWldKbXInamxMcW1MUSo0X2lOaHItYmNKX1xjLldIIitwNkxPMSsxJjcnVzZaMjotUUZZNTksJ1M3PnE+QUhKLDdlYE8uKSJeSCtPPiNYPitLX0VPTjA+XWAqVHVfYUhOLlVEVnIhZWFNWE8lZUZDV09TMGNcVGE8UjdIZS9kXkhiVW5OMmZcJk9HSUdWb3QsMDhmODA8K3NSSzJtXkZ+PmVuZHN0cmVhbQplbmRvYmoKOCAwIG9iago8PAovQ29udGVudHMgMTkgMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXSAvWE9iamVjdCA8PAovRm9ybVhvYi4yYTg4YWM2NzEyYzljYTcyN2U0YWQ4ZjI1YmNiZGFlMiA3IDAgUiAvRm9ybVhvYi42YmU0MGVlYWFmMGQ4MWQxODc1MGRjMGE3ZjlkYzIxMyA1IDAgUiAvRm9ybVhvYi5lNWMzMTQ3ZDE4Zjk4NjdhNGYxMDhkM2E1NWEyNDhiNCA2IDAgUgo+Pgo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKOSAwIG9iago8PAovQSA8PAovUyAvVVJJIC9UeXBlIC9BY3Rpb24gL1VSSSAoaHR0cHM6Ly9leGFtcGxlLmNvbS9pbnZvaWNlcy9JTlYtMjA0OCkKPj4gL0JvcmRlciBbIDAgMCAwIF0gL1JlY3QgWyA0NSA1NzUgMzAwIDU5MCBdIC9TdWJ0eXBlIC9MaW5rIC9UeXBlIC9Bbm5vdAo+PgplbmRvYmoKMTAgMCBvYmoKPDwKL0MgWyAuODMgLjg5IC45NSBdIC9Db250ZW50cyAoVG90YWwgaGlnaGxpZ2h0ZWQgZm9yIHJldmlldykgL1F1YWRQb2ludHMgWyA0MCA1NzkgNTQwIDU3OSA0MCA1NTUgNTQwIDU1NSBdIC9SZWN0IFsgNDAgNTU1IDU0MCA1NzkgXSAvU3VidHlwZSAvSGlnaGxpZ2h0IC9UeXBlIC9Bbm5vdAo+PgplbmRvYmoKMTEgMCBvYmoKPDwKL0MgWyAwIDAgMCBdIC9Db250ZW50cyAoVmVyaWZ5IHRoZSBoaWdobGlnaHRlZCB0b3RhbCkgL1F1YWRQb2ludHMgWyA1MjAgNTI1IDU0MCA1MjUgNTIwIDU0NSA1NDAgNTQ1IF0gL1JlY3QgWyA1MjAgNTI1IDU0MCA1NDUgXSAvU3VidHlwZSAvVGV4dCAvVHlwZSAvQW5ub3QKPj4KZW5kb2JqCjEyIDAgb2JqCjw8Ci9Bbm5vdHMgWyA5IDAgUiAxMCAwIFIgMTEgMCBSIF0gL0NvbnRlbnRzIDIwIDAgUiAvTWVkaWFCb3ggWyAwIDAgNjEyIDc5MiBdIC9QYXJlbnQgMTcgMCBSIC9SZXNvdXJjZXMgPDwKL0ZvbnQgMSAwIFIgL1Byb2NTZXQgWyAvUERGIC9UZXh0IC9JbWFnZUIgL0ltYWdlQyAvSW1hZ2VJIF0KPj4gL1JvdGF0ZSAwIAogIC9UcmFucyA8PAoKPj4gL1R5cGUgL1BhZ2UKPj4KZW5kb2JqCjEzIDAgb2JqCjw8Ci9Db250ZW50cyAyMSAwIFIgL01lZGlhQm94IFsgMCAwIDYxMiA3OTIgXSAvUGFyZW50IDE3IDAgUiAvUmVzb3VyY2VzIDw8Ci9Gb250IDEgMCBSIC9Qcm9jU2V0IFsgL1BERiAvVGV4dCAvSW1hZ2VCIC9JbWFnZUMgL0ltYWdlSSBdCj4+IC9Sb3RhdGUgMCAvVHJhbnMgPDwKCj4+IAogIC9UeXBlIC9QYWdlCj4+CmVuZG9iagoxNCAwIG9iago8PAovQ29udGVudHMgMjIgMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXQo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKMTUgMCBvYmoKPDwKL1BhZ2VNb2RlIC9Vc2VOb25lIC9QYWdlcyAxNyAwIFIgL1R5cGUgL0NhdGFsb2cKPj4KZW5kb2JqCjE2IDAgb2JqCjw8Ci9BdXRob3IgKExpdGVMTE0gT0NSIGZpeHR1cmUgZ2VuZXJhdG9yKSAvQ3JlYXRpb25EYXRlIChEOjIwMDAwMTAxMDAwMDAwKzAwJzAwJykgL0NyZWF0b3IgKGFub255bW91cykgL0tleXdvcmRzIChPQ1IsIGludm9pY2UsIHRhYmxlLCBmaWd1cmUsIGFubm90YXRpb24pIC9Nb2REYXRlIChEOjIwMDAwMTAxMDAwMDAwKzAwJzAwJykgL1Byb2R1Y2VyIChSZXBvcnRMYWIgUERGIExpYnJhcnkgLSBcKG9wZW5zb3VyY2VcKSkgCiAgL1N1YmplY3QgKFNlbWFudGljIE9DUiBjb3ZlcmFnZSBmb3IgdGFibGVzLCBmaWd1cmVzLCBhbm5vdGF0aW9ucywgYW5kIG1ldGFkYXRhKSAvVGl0bGUgKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgL1RyYXBwZWQgL0ZhbHNlCj4+CmVuZG9iagoxNyAwIG9iago8PAovQ291bnQgNSAvS2lkcyBbIDQgMCBSIDggMCBSIDEyIDAgUiAxMyAwIFIgMTQgMCBSIF0gL1R5cGUgL1BhZ2VzCj4+CmVuZG9iagoxOCAwIG9iago8PAovTGVuZ3RoIDIyODAKPj4Kc3RyZWFtCjEgMCAwIDEgMCAwIGNtICBCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVAowIDAgMCByZwpCVCAvRjEgMTEgVGYgMTMuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc3MCBUbSAoUXVhcnRlcmx5IE9wZXJhdGlvbnMgUmVwb3J0KSBUaiBUKiBFVApCVCAvRjIgMTYgVGYgMTkuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc0NSBUbSAoSW52b2ljZSBTdW1tYXJ5IGFuZCBMaW5lIEl0ZW1zKSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgMSBvZiA1KSBUaiBUKiBFVApuIDQ1IDYyNSBtIDQ1IDczMCBsIFMKbiAyNDUgNjI1IG0gMjQ1IDczMCBsIFMKbiA0MDUgNjI1IG0gNDA1IDczMCBsIFMKbiA1NjUgNjI1IG0gNTY1IDczMCBsIFMKbiA0NSA3MzAgbSA1NjUgNzMwIGwgUwpuIDQ1IDY5NSBtIDU2NSA2OTUgbCBTCm4gNDUgNjYwIG0gNTY1IDY2MCBsIFMKbiA0NSA2MjUgbSA1NjUgNjI1IGwgUwpCVCAxIDAgMCAxIDU1IDcwNyBUbSAoSXRlbSkgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNzA3IFRtIChRdWFudGl0eSkgVGogVCogRVQKQlQgMSAwIDAgMSA0MTUgNzA3IFRtIChBbW91bnQpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNTUgNjcyIFRtIChEb2N1bWVudCBhbmFseXNpcykgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNjcyIFRtICgyKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA2NzIgVG0gKDEyMC4wMCkgVGogVCogRVQKQlQgMSAwIDAgMSA1NSA2MzcgVG0gKE9DUiB2ZXJpZmljYXRpb24pIFRqIFQqIEVUCkJUIDEgMCAwIDEgMjU1IDYzNyBUbSAoMSkgVGogVCogRVQKQlQgMSAwIDAgMSA0MTUgNjM3IFRtICg4MC4wMCkgVGogVCogRVQKbiA0NSA0OTUgbSA0NSA2MDAgbCBTCm4gMjQ1IDQ5NSBtIDI0NSA2MDAgbCBTCm4gNDA1IDQ5NSBtIDQwNSA2MDAgbCBTCm4gNTY1IDQ5NSBtIDU2NSA2MDAgbCBTCm4gNDUgNjAwIG0gNTY1IDYwMCBsIFMKbiA0NSA1NjUgbSA1NjUgNTY1IGwgUwpuIDQ1IDUzMCBtIDU2NSA1MzAgbCBTCm4gNDUgNDk1IG0gNTY1IDQ5NSBsIFMKQlQgMSAwIDAgMSA1NSA1NzcgVG0gKEl0ZW0gY29udGludWVkKSBUaiBUKiBFVApCVCAxIDAgMCAxIDI1NSA1NzcgVG0gKFF1YW50aXR5KSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA1NzcgVG0gKEFtb3VudCkgVGogVCogRVQKQlQgMSAwIDAgMSA1NSA1NDIgVG0gKEZpeHR1cmUgdmFsaWRhdGlvbikgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNTQyIFRtICgzKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA1NDIgVG0gKDQ1LjAwKSBUaiBUKiBFVApCVCAxIDAgMCAxIDU1IDUwNyBUbSAoUHJvdmlkZXIgcmV2aWV3KSBUaiBUKiBFVApCVCAxIDAgMCAxIDI1NSA1MDcgVG0gKDEpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDE1IDUwNyBUbSAoMjUuMDApIFRqIFQqIEVUCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiAxLjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiAxLjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiAxLjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiAxLjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiAxLjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiAxLjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiAxLjc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiAxLjg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKMTkgMCBvYmoKPDwKL0xlbmd0aCAxNzIzCj4+CnN0cmVhbQoxIDAgMCAxIDAgMCBjbSAgQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKMCAwIDAgcmcKQlQgL0YxIDExIFRmIDEzLjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NzAgVG0gKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgVGogVCogRVQKQlQgL0YyIDE2IFRmIDE5LjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NDUgVG0gKFJldmVudWUgQ2hhcnQgYW5kIEZvcm11bGEgUmV2aWV3KSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgMiBvZiA1KSBUaiBUKiBFVAouMiAuNDUwOTggLjg1MDk4IHJnCm4gNzAgNjEwIDY1IDcwIHJlIGYqCm4gMTcwIDYxMCA2NSAxMTUgcmUgZioKbiAyNzAgNjEwIDY1IDkwIHJlIGYqCm4gMzcwIDYxMCA2NSAxMzAgcmUgZioKMCAwIDAgcmcKQlQgMSAwIDAgMSA5MCA1OTAgVG0gKFExKSBUaiBUKiBFVApCVCAxIDAgMCAxIDE5MCA1OTAgVG0gKFEyKSBUaiBUKiBFVApCVCAxIDAgMCAxIDI5MCA1OTAgVG0gKFEzKSBUaiBUKiBFVApCVCAxIDAgMCAxIDM5MCA1OTAgVG0gKFE0KSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDU1MCBUbSAoRm9ybXVsYTogZ3Jvc3MgbWFyZ2luID0gXChyZXZlbnVlIC0gY29zdFwpIC8gcmV2ZW51ZSkgVGogVCogRVQKcQo3MCAwIDAgNzAgNDU1IDY1NSBjbQovRm9ybVhvYi42YmU0MGVlYWFmMGQ4MWQxODc1MGRjMGE3ZjlkYzIxMyBEbwpRCnEKNzAgMCAwIDYyIDQ1NSA1NjUgY20KL0Zvcm1Yb2IuZTVjMzE0N2QxOGY5ODY3YTRmMTA4ZDNhNTVhMjQ4YjQgRG8KUQpxCjQ1IDAgMCA0NSA0NTUgNTAwIGNtCi9Gb3JtWG9iLjJhODhhYzY3MTJjOWNhNzI3ZTRhZDhmMjViY2JkYWUyIERvClEKQlQgL0YxIDEwIFRmIDEyIFRMIEVUCkJUIDEgMCAwIDEgNDUgNDcyIFRtIChTZWN0aW9uIDIuMTogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNDQ0IFRtIChTZWN0aW9uIDIuMjogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNDE2IFRtIChTZWN0aW9uIDIuMzogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzg4IFRtIChTZWN0aW9uIDIuNDogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzYwIFRtIChTZWN0aW9uIDIuNTogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzMyIFRtIChTZWN0aW9uIDIuNjogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzA0IFRtIChTZWN0aW9uIDIuNzogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMjc2IFRtIChTZWN0aW9uIDIuODogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCiAKZW5kc3RyZWFtCmVuZG9iagoyMCAwIG9iago8PAovTGVuZ3RoIDI0NjYKPj4Kc3RyZWFtCjEgMCAwIDEgMCAwIGNtICBCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVAowIDAgMCByZwpCVCAvRjEgMTEgVGYgMTMuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc3MCBUbSAoUXVhcnRlcmx5IE9wZXJhdGlvbnMgUmVwb3J0KSBUaiBUKiBFVApCVCAvRjIgMTYgVGYgMTkuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc0NSBUbSAoS2V5IFZhbHVlcywgTGluaywgSGlnaGxpZ2h0LCBhbmQgQ29tbWVudCkgVGogVCogRVQKQlQgL0YxIDkgVGYgMTAuOCBUTCBFVApCVCAxIDAgMCAxIDQ1IDMwIFRtIChDb25maWRlbnRpYWwgfCBQYWdlIDMgb2YgNSkgVGogVCogRVQKQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKQlQgMSAwIDAgMSA0NSA3MDAgVG0gKEludm9pY2UgTnVtYmVyOiBJTlYtMjA0OCkgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA2NzUgVG0gKFB1cmNoYXNlIE9yZGVyOiBQTy00MDk2KSBUaiBUKiBFVAouOTQ5MDIgLjkwMTk2MSAuMzUyOTQxIHJnCm4gNDAgNTU1IDUwMCAyNCByZSBmKgowIDAgMCByZwpCVCAxIDAgMCAxIDQ1IDU2MCBUbSAoSGlnaGxpZ2h0ZWQgdG90YWwgcmVxdWlyaW5nIHJldmlldykgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA1MzAgVG0gKFJldmlld2VyIGNvbW1lbnQ6IHZlcmlmeSB0aGUgaGlnaGxpZ2h0ZWQgdG90YWwgYmVmb3JlIGFwcHJvdmFsKSBUaiBUKiBFVAoxIDAgMCByZwpCVCAxIDAgMCAxIDQ1IDQ5NSBUbSAoUmV2aXNlZCB0b3RhbDogMjQ1LjAwKSBUaiBUKiBFVApuIDQ1IDUwMSBtIDE1MCA1MDEgbCBTCjAgMCAwIHJnCkJUIDEgMCAwIDEgNDUgNTc1IFRtIChodHRwczovL2V4YW1wbGUuY29tL2ludm9pY2VzL0lOVi0yMDQ4KSBUaiBUKiBFVApxCjEgMCAwIDEgOTAgMTMwIGNtCm4gMTggMCAyLjQgNzAgcmUgZioKbiAyMS42IDAgMS4yIDcwIHJlIGYqCm4gMjUuMiAwIDMuNiA3MCByZSBmKgpuIDMxLjIgMCAzLjYgNzAgcmUgZioKbiAzOC40IDAgMi40IDcwIHJlIGYqCm4gNDIgMCAxLjIgNzAgcmUgZioKbiA0NC40IDAgMi40IDcwIHJlIGYqCm4gNDkuMiAwIDIuNCA3MCByZSBmKgpuIDUyLjggMCAyLjQgNzAgcmUgZioKbiA1Ny42IDAgMy42IDcwIHJlIGYqCm4gNjIuNCAwIDIuNCA3MCByZSBmKgpuIDY2IDAgMy42IDcwIHJlIGYqCm4gNzAuOCAwIDIuNCA3MCByZSBmKgpuIDc2LjggMCAxLjIgNzAgcmUgZioKbiA4MS42IDAgMS4yIDcwIHJlIGYqCm4gODQgMCAzLjYgNzAgcmUgZioKbiA4OC44IDAgMi40IDcwIHJlIGYqCm4gOTIuNCAwIDMuNiA3MCByZSBmKgpuIDk3LjIgMCAxLjIgNzAgcmUgZioKbiA5OS42IDAgMy42IDcwIHJlIGYqCm4gMTA0LjQgMCAyLjQgNzAgcmUgZioKbiAxMTAuNCAwIDEuMiA3MCByZSBmKgpuIDExMi44IDAgNC44IDcwIHJlIGYqCm4gMTE4LjggMCAzLjYgNzAgcmUgZioKbiAxMjMuNiAwIDMuNiA3MCByZSBmKgpuIDEyOC40IDAgMi40IDcwIHJlIGYqCm4gMTMyIDAgMy42IDcwIHJlIGYqCm4gMTM2LjggMCAyLjQgNzAgcmUgZioKbiAxNDEuNiAwIDEuMiA3MCByZSBmKgpuIDE0NS4yIDAgMy42IDcwIHJlIGYqCm4gMTUwIDAgMi40IDcwIHJlIGYqCm4gMTU2IDAgMy42IDcwIHJlIGYqCm4gMTYwLjggMCAxLjIgNzAgcmUgZioKbiAxNjMuMiAwIDIuNCA3MCByZSBmKgpRCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiAzLjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiAzLjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiAzLjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiAzLjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiAzLjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiAzLjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiAzLjc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiAzLjg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKMjEgMCBvYmoKPDwKL0xlbmd0aCAxNDk0Cj4+CnN0cmVhbQoxIDAgMCAxIDAgMCBjbSAgQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKMCAwIDAgcmcKQlQgL0YxIDExIFRmIDEzLjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NzAgVG0gKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgVGogVCogRVQKQlQgL0YyIDE2IFRmIDE5LjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NDUgVG0gKEFwcHJvdmFsIFNpZ25hdHVyZSBhbmQgV2F0ZXJtYXJrKSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgNCBvZiA1KSBUaiBUKiBFVApxCi44Mjc0NTEgLjgyNzQ1MSAuODI3NDUxIHJnCkJUIC9GMiA1NCBUZiA2NC44IFRMIEVUCi45MDYzMDggLjQyMjYxOCAtMC40MjI2MTggLjkwNjMwOCAxMTAgMzkwIGNtCkJUIDEgMCAwIDEgMCAwIFRtIChEUkFGVCkgVGogVCogRVQKUQowIDAgMCByZwpCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVApCVCAxIDAgMCAxIDQ1IDYzNSBUbSAoQXBwcm92ZWQgYnk6IEpvcmRhbiBMZWUpIFRqIFQqIEVUCm4gNDUgNjEwIG0gMzEwIDYxMCBsIFMKbiA1NSA1OTUgbSA3NSA2MjUgMTEyIDYwMiAxNTUgNjAwIGMgUwpCVCAxIDAgMCAxIDQ1IDU4MCBUbSAoU2lnbmF0dXJlKSBUaiBUKiBFVApCVCAvRjEgMTAgVGYgMTIgVEwgRVQKQlQgMSAwIDAgMSA0NSA0NzIgVG0gKFNlY3Rpb24gNC4xOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA0NDQgVG0gKFNlY3Rpb24gNC4yOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA0MTYgVG0gKFNlY3Rpb24gNC4zOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzODggVG0gKFNlY3Rpb24gNC40OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzNjAgVG0gKFNlY3Rpb24gNC41OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzMzIgVG0gKFNlY3Rpb24gNC42OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzMDQgVG0gKFNlY3Rpb24gNC43OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAyNzYgVG0gKFNlY3Rpb24gNC44OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKIAplbmRzdHJlYW0KZW5kb2JqCjIyIDAgb2JqCjw8Ci9MZW5ndGggMTMyNwo+PgpzdHJlYW0KMSAwIDAgMSAwIDAgY20gIEJUIC9GMSAxMiBUZiAxNC40IFRMIEVUCjAgMCAwIHJnCkJUIC9GMSAxMSBUZiAxMy4yIFRMIEVUCkJUIDEgMCAwIDEgNDUgNzcwIFRtIChRdWFydGVybHkgT3BlcmF0aW9ucyBSZXBvcnQpIFRqIFQqIEVUCkJUIC9GMiAxNiBUZiAxOS4yIFRMIEVUCkJUIDEgMCAwIDEgNDUgNzQ1IFRtIChBcHBlbmRpeCB3aXRoIFNlY3Rpb24gQm91bmRhcmllcykgVGogVCogRVQKQlQgL0YxIDkgVGYgMTAuOCBUTCBFVApCVCAxIDAgMCAxIDQ1IDMwIFRtIChDb25maWRlbnRpYWwgfCBQYWdlIDUgb2YgNSkgVGogVCogRVQKQlQgL0YyIDE0IFRmIDE2LjggVEwgRVQKQlQgMSAwIDAgMSA0NSA3MDAgVG0gKDEuIFNjb3BlKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDY1MCBUbSAoMi4gRmluZGluZ3MpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNjAwIFRtICgzLiBSZWNvbW1lbmRhdGlvbnMpIFRqIFQqIEVUCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiA1LjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiA1LjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiA1LjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiA1LjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiA1LjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiA1LjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiA1Ljc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiA1Ljg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDIzCjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDA2MSAwMDAwMCBuIAowMDAwMDAwMTAyIDAwMDAwIG4gCjAwMDAwMDAyMDkgMDAwMDAgbiAKMDAwMDAwMDMyMSAwMDAwMCBuIAowMDAwMDAwNTE2IDAwMDAwIG4gCjAwMDAwMDYzMjIgMDAwMDAgbiAKMDAwMDAxMjgyNCAwMDAwMCBuIAowMDAwMDE0MTQyIDAwMDAwIG4gCjAwMDAwMTQ0OTYgMDAwMDAgbiAKMDAwMDAxNDY2NCAwMDAwMCBuIAowMDAwMDE0ODUwIDAwMDAwIG4gCjAwMDAwMTUwMjggMDAwMDAgbiAKMDAwMDAxNTI1NiAwMDAwMCBuIAowMDAwMDE1NDUyIDAwMDAwIG4gCjAwMDAwMTU2NDggMDAwMDAgbiAKMDAwMDAxNTcxOCAwMDAwMCBuIAowMDAwMDE2MTE1IDAwMDAwIG4gCjAwMDAwMTYyMDIgMDAwMDAgbiAKMDAwMDAxODUzNCAwMDAwMCBuIAowMDAwMDIwMzA5IDAwMDAwIG4gCjAwMDAwMjI4MjcgMDAwMDAgbiAKMDAwMDAyNDM3MyAwMDAwMCBuIAp0cmFpbGVyCjw8Ci9JRCAKWzwyZjUwODkzYTFlYWZmMTExOWMwNDcwZmM0YzU3ZTI0Nz48MmY1MDg5M2ExZWFmZjExMTljMDQ3MGZjNGM1N2UyNDc+XQolIFJlcG9ydExhYiBnZW5lcmF0ZWQgUERGIGRvY3VtZW50IC0tIGRpZ2VzdCAob3BlbnNvdXJjZSkKCi9JbmZvIDE2IDAgUgovUm9vdCAxNSAwIFIKL1NpemUgMjMKPj4Kc3RhcnR4cmVmCjI1NzUyCiUlRU9GCg== + type: document_url + model: mistral/invalid-ocr-model-for-parity + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/1f38e1eab857c22107df833aa63b119bced6537f45cf08b27cbec08bb8f45951.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/1f38e1eab857c22107df833aa63b119bced6537f45cf08b27cbec08bb8f45951.yaml new file mode 100644 index 00000000000..8200c885e60 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/1f38e1eab857c22107df833aa63b119bced6537f45cf08b27cbec08bb8f45951.yaml @@ -0,0 +1,69 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"pages":[0]}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d35d1fbbebe5-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:14 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-5685-747f-b76d-dab242ea7512 + x-envoy-upstream-service-time: + - '179' + x-kong-proxy-latency: + - '12' + x-kong-request-id: + - 01a05e89-5685-747f-b76d-dab242ea7512 + x-kong-upstream-latency: + - '180' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '58' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:14.385374+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + model: mistral/mistral-ocr-latest + pages: + - 0 + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/38dbc05d8508355879de894ecc0f46da887570de720a142e9df297bdd280608d.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/38dbc05d8508355879de894ecc0f46da887570de720a142e9df297bdd280608d.yaml new file mode 100644 index 00000000000..a9fd2d0fbec --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/38dbc05d8508355879de894ecc0f46da887570de720a142e9df297bdd280608d.yaml @@ -0,0 +1,85 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"document_annotation_format":{"type":"json_schema","json_schema":{"name":"document_title","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}},"document_annotation_prompt":"Extract + the visible title"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":"{\"title\": + \"invoice 123\"}","usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d376581f74f9-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:18 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-664a-740d-a112-0a1183c302b3 + x-envoy-upstream-service-time: + - '402' + x-kong-proxy-latency: + - '20' + x-kong-request-id: + - 01a05e89-664a-740d-a112-0a1183c302b3 + x-kong-upstream-latency: + - '403' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '52' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:18.915814+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + document_annotation_format: + json_schema: + description: Extract the visible document fields + name: document_title + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + document_annotation_prompt: Extract the visible title + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/3d5f42192461789f547b42c8f1b625aa688e6cbd69e01f94d1f3205ef41ab830.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/3d5f42192461789f547b42c8f1b625aa688e6cbd69e01f94d1f3205ef41ab830.yaml new file mode 100644 index 00000000000..a471b8f7bae --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/3d5f42192461789f547b42c8f1b625aa688e6cbd69e01f94d1f3205ef41ab830.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"extract_header":true}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d37c9944d8a7-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:19 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-6a3a-76c7-85b8-d40a7156155e + x-envoy-upstream-service-time: + - '230' + x-kong-proxy-latency: + - '16' + x-kong-request-id: + - 01a05e89-6a3a-76c7-85b8-d40a7156155e + x-kong-upstream-latency: + - '230' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '51' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:19.420285+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + extract_header: true + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/54ce0991a21ea668e2373d2b05801606b7039583f55a0f85bfe38c7c15712adf.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/54ce0991a21ea668e2373d2b05801606b7039583f55a0f85bfe38c7c15712adf.yaml new file mode 100644 index 00000000000..90a4abd5574 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/54ce0991a21ea668e2373d2b05801606b7039583f55a0f85bfe38c7c15712adf.yaml @@ -0,0 +1,67 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d356fb3698ce-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:13 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-52b4-7e7b-983e-5bc4e5469b39 + x-envoy-upstream-service-time: + - '554' + x-kong-proxy-latency: + - '13' + x-kong-request-id: + - 01a05e89-52b4-7e7b-983e-5bc4e5469b39 + x-kong-upstream-latency: + - '557' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '59' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:13.880852+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/7f7dc004625cb90e3f4bd2f473fda532390140ade4e10b5bb07570ca7b372499.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/7f7dc004625cb90e3f4bd2f473fda532390140ade4e10b5bb07570ca7b372499.yaml new file mode 100644 index 00000000000..748948e5e8c --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/7f7dc004625cb90e3f4bd2f473fda532390140ade4e10b5bb07570ca7b372499.yaml @@ -0,0 +1,82 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"bbox_annotation_format":{"type":"json_schema","json_schema":{"name":"bounding_boxes","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d369be838fc5-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:16 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-5e6e-70ac-890d-c0110a5e81af + x-envoy-upstream-service-time: + - '210' + x-kong-proxy-latency: + - '17' + x-kong-request-id: + - 01a05e89-5e6e-70ac-890d-c0110a5e81af + x-kong-upstream-latency: + - '212' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '54' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:16.402593+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + bbox_annotation_format: + json_schema: + description: Extract the visible document fields + name: bounding_boxes + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/99d9d5ac0449213a562a36163ee598ef9e6df1813bee21137e88bcc14ce7a473.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/99d9d5ac0449213a562a36163ee598ef9e6df1813bee21137e88bcc14ce7a473.yaml new file mode 100644 index 00000000000..80c9a75578a --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/99d9d5ac0449213a562a36163ee598ef9e6df1813bee21137e88bcc14ce7a473.yaml @@ -0,0 +1,92 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"document_url","document_url":"data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg=="},"pages":[0],"include_image_base64":true,"image_limit":1,"image_min_size":300,"bbox_annotation_format":{"type":"json_schema","json_schema":{"name":"bounding_boxes","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}},"extract_header":true,"extract_footer":false,"table_format":"markdown","confidence_scores_granularity":"page","include_blocks":false,"id":"case-1"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"Test PDF File","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":93,"height":1023,"width":791},"confidence_scores":{"word_confidence_scores":[],"average_page_confidence_score":0.9376229744322936,"minimum_page_confidence_score":0.22590550796036835},"blocks":null}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":589}}' + headers: + CF-RAY: + - a346d39c1fe5cf12-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:24 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-7deb-72b4-976d-4c244b056782 + x-envoy-upstream-service-time: + - '373' + x-kong-proxy-latency: + - '17' + x-kong-request-id: + - 01a05e89-7deb-72b4-976d-4c244b056782 + x-kong-upstream-latency: + - '373' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '45' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:24.951956+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + bbox_annotation_format: + json_schema: + description: Extract the visible document fields + name: bounding_boxes + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + confidence_scores_granularity: page + contract: mistral + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + extract_footer: false + extract_header: true + id: case-1 + image_limit: 1 + image_min_size: 300 + include_blocks: false + include_image_base64: true + model: mistral/mistral-ocr-latest + pages: + - 0 + table_format: markdown + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/9b9fb2f3a7eb8ca1e2d128ea96b358dc7c9b0eafc64e7e2516d46e290fc93092.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/9b9fb2f3a7eb8ca1e2d128ea96b358dc7c9b0eafc64e7e2516d46e290fc93092.yaml new file mode 100644 index 00000000000..847bac3f431 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/9b9fb2f3a7eb8ca1e2d128ea96b358dc7c9b0eafc64e7e2516d46e290fc93092.yaml @@ -0,0 +1,83 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"document_annotation_format":{"type":"json_schema","json_schema":{"name":"document_title","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":"{\"title\": + \"Invoice_123\"}","usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d36cd82f15ba-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:17 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-6060-71cb-9e35-66ab873b99f3 + x-envoy-upstream-service-time: + - '888' + x-kong-proxy-latency: + - '12' + x-kong-request-id: + - 01a05e89-6060-71cb-9e35-66ab873b99f3 + x-kong-upstream-latency: + - '889' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '53' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:17.908846+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + document_annotation_format: + json_schema: + description: Extract the visible document fields + name: document_title + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/a8693352d9ce0d7d120b2351d2257cd562048fc6a993cba093fe86a8937d104b.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/a8693352d9ce0d7d120b2351d2257cd562048fc6a993cba093fe86a8937d104b.yaml new file mode 100644 index 00000000000..3b5f8bb85e1 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/a8693352d9ce0d7d120b2351d2257cd562048fc6a993cba093fe86a8937d104b.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"include_image_base64":true}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d360593c138a-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:14 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-588e-7e32-a54b-f816bc6c1dc4 + x-envoy-upstream-service-time: + - '242' + x-kong-proxy-latency: + - '16' + x-kong-request-id: + - 01a05e89-588e-7e32-a54b-f816bc6c1dc4 + x-kong-upstream-latency: + - '242' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '57' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:14.889667+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + include_image_base64: true + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/b19fdc140375a4eadfd6eb4e4cbe88e27651580bac1e145ad352728b1a0acd31.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/b19fdc140375a4eadfd6eb4e4cbe88e27651580bac1e145ad352728b1a0acd31.yaml new file mode 100644 index 00000000000..acabdaf2b5e --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/b19fdc140375a4eadfd6eb4e4cbe88e27651580bac1e145ad352728b1a0acd31.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"image_min_size":300}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d36688d203c2-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:15 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-5c60-769b-abf3-44dcefdf5282 + x-envoy-upstream-service-time: + - '272' + x-kong-proxy-latency: + - '17' + x-kong-request-id: + - 01a05e89-5c60-769b-abf3-44dcefdf5282 + x-kong-upstream-latency: + - '273' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '55' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:15.898087+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + image_min_size: 300 + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/bdb616d21dece89481debccb9358047b28006b5cbd4486d54c6fe89070e7a671.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/bdb616d21dece89481debccb9358047b28006b5cbd4486d54c6fe89070e7a671.yaml new file mode 100644 index 00000000000..ed060cd05ca --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/bdb616d21dece89481debccb9358047b28006b5cbd4486d54c6fe89070e7a671.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"id":"case-1"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d398da783ad4-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:23 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-7bdc-74c5-b8eb-15fbd25b6cfd + x-envoy-upstream-service-time: + - '182' + x-kong-proxy-latency: + - '22' + x-kong-request-id: + - 01a05e89-7bdc-74c5-b8eb-15fbd25b6cfd + x-kong-upstream-latency: + - '183' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '46' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:23.946966+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + id: case-1 + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c3066e2f2f964725b71ae891821d77447359bba8c29d6b31e650255298271d47.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c3066e2f2f964725b71ae891821d77447359bba8c29d6b31e650255298271d47.yaml new file mode 100644 index 00000000000..22d01018046 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c3066e2f2f964725b71ae891821d77447359bba8c29d6b31e650255298271d47.yaml @@ -0,0 +1,67 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"include_blocks":false}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":null}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d392a8432af7-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:22 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-7802-768c-bf57-9021a209ab48 + x-envoy-upstream-service-time: + - '213' + x-kong-proxy-latency: + - '17' + x-kong-request-id: + - 01a05e89-7802-768c-bf57-9021a209ab48 + x-kong-upstream-latency: + - '214' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '47' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:23.442341+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + include_blocks: false + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c57601c90e9b289e506cdc818523bba41a29b707d0cfafd74986e6ad8c2ed900.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c57601c90e9b289e506cdc818523bba41a29b707d0cfafd74986e6ad8c2ed900.yaml new file mode 100644 index 00000000000..9d1a1e55835 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c57601c90e9b289e506cdc818523bba41a29b707d0cfafd74986e6ad8c2ed900.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"confidence_scores_granularity":"page"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":{"word_confidence_scores":[],"average_page_confidence_score":0.90845564554897,"minimum_page_confidence_score":0.16168208839823475},"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d38c59681749-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:22 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-7411-7d2c-9e43-01320e4cad1c + x-envoy-upstream-service-time: + - '329' + x-kong-proxy-latency: + - '14' + x-kong-request-id: + - 01a05e89-7411-7d2c-9e43-01320e4cad1c + x-kong-upstream-latency: + - '330' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '48' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:22.437385+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + confidence_scores_granularity: page + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c5d33cb2b20fb76c9543ff57dc9b3b7dc37c5bf5dfd47e873c4cc695d6d7461f.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c5d33cb2b20fb76c9543ff57dc9b3b7dc37c5bf5dfd47e873c4cc695d6d7461f.yaml new file mode 100644 index 00000000000..7afaa689756 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c5d33cb2b20fb76c9543ff57dc9b3b7dc37c5bf5dfd47e873c4cc695d6d7461f.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"extract_footer":true}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d37fde0f1703-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:20 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-6c42-7ea1-b08b-07cae50734e0 + x-envoy-upstream-service-time: + - '523' + x-kong-proxy-latency: + - '13' + x-kong-request-id: + - 01a05e89-6c42-7ea1-b08b-07cae50734e0 + x-kong-upstream-latency: + - '525' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '50' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:20.425444+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + extract_footer: true + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c6f0f158205be7fc00bd962e341689c3583b09694a7ac086e4707be49b35eb06.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c6f0f158205be7fc00bd962e341689c3583b09694a7ac086e4707be49b35eb06.yaml new file mode 100644 index 00000000000..09c39780583 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c6f0f158205be7fc00bd962e341689c3583b09694a7ac086e4707be49b35eb06.yaml @@ -0,0 +1,83 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"document_url","document_url":"data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg=="},"document_annotation_format":{"type":"json_schema","json_schema":{"name":"document_title","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"Test PDF File","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":93,"height":1023,"width":791},"confidence_scores":null,"blocks":[{"top_left_x":126,"top_left_y":104,"bottom_right_x":229,"bottom_right_y":127,"content":"Test + PDF File","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":"{\"title\": + \"Test_PDF_File\"}","usage_info":{"pages_processed":1,"doc_size_bytes":589}}' + headers: + CF-RAY: + - a346d3a25d0cccb8-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:25 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-81d5-79f8-914d-e3863af2d24b + x-envoy-upstream-service-time: + - '448' + x-kong-proxy-latency: + - '15' + x-kong-request-id: + - 01a05e89-81d5-79f8-914d-e3863af2d24b + x-kong-upstream-latency: + - '449' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '44' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:25.959391+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + document_annotation_format: + json_schema: + description: Extract the visible document fields + name: document_title + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/cea250810b49cd17251ee8c036eae74dd363ddda0d6677155afbd145d0dc3c36.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/cea250810b49cd17251ee8c036eae74dd363ddda0d6677155afbd145d0dc3c36.yaml new file mode 100644 index 00000000000..85d37a96634 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/cea250810b49cd17251ee8c036eae74dd363ddda0d6677155afbd145d0dc3c36.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"table_format":"markdown"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d385ffbda0f2-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:21 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-7015-7a8d-a18c-37b9ceacbe77 + x-envoy-upstream-service-time: + - '337' + x-kong-proxy-latency: + - '21' + x-kong-request-id: + - 01a05e89-7015-7a8d-a18c-37b9ceacbe77 + x-kong-upstream-latency: + - '338' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '49' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:21.430472+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + model: mistral/mistral-ocr-latest + table_format: markdown + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/d80a6ccca02922ba9094426235a6ab165f1866eb074a8c079273eef5df512985.yaml b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/d80a6ccca02922ba9094426235a6ab165f1866eb074a8c079273eef5df512985.yaml new file mode 100644 index 00000000000..5e8fe1a1cdc --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/d80a6ccca02922ba9094426235a6ab165f1866eb074a8c079273eef5df512985.yaml @@ -0,0 +1,105 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"pages":[0],"include_image_base64":false,"image_min_size":300,"bbox_annotation_format":{"type":"json_schema","json_schema":{"name":"bounding_boxes","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}},"document_annotation_format":{"type":"json_schema","json_schema":{"name":"document_title","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}},"extract_header":true,"table_format":"markdown","include_blocks":true}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":"{\"title\": + \"Invoice_123\"}","usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d3a89a85f953-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:26 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-85bd-7075-af53-8e341b35a63e + x-envoy-upstream-service-time: + - '432' + x-kong-proxy-latency: + - '13' + x-kong-request-id: + - 01a05e89-85bd-7075-af53-8e341b35a63e + x-kong-upstream-latency: + - '433' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '43' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:26.965078+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + bbox_annotation_format: + json_schema: + description: Extract the visible document fields + name: bounding_boxes + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + document_annotation_format: + json_schema: + description: Extract the visible document fields + name: document_title + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + extract_header: true + image_min_size: 300 + include_blocks: true + include_image_base64: false + model: mistral/mistral-ocr-latest + pages: + - 0 + table_format: markdown + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-legacy/09cc2f0155ffdc3d25abff96797030105ad107ae3400532b20ad70ebbd021954.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-legacy/09cc2f0155ffdc3d25abff96797030105ad107ae3400532b20ad70ebbd021954.yaml new file mode 100644 index 00000000000..36cb1ff8ce1 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-legacy/09cc2f0155ffdc3d25abff96797030105ad107ae3400532b20ad70ebbd021954.yaml @@ -0,0 +1,70 @@ +interactions: +- request: + body: "--3d54cb2e388c04dbfc172c1497aaa396\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--3d54cb2e388c04dbfc172c1497aaa396--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=3d54cb2e388c04dbfc172c1497aaa396 + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://45d3cbbc-4d77-4967-8941-935b0c4a0493.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:58 GMT + status: + code: 200 + message: '' +- request: + body: '{"document_url":"reducto://45d3cbbc-4d77-4967-8941-935b0c4a0493.pdf"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"262ed683-ecd8-40b9-a568-f7b7014854c9","duration":2.4146597385406494,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/a91f568b-6e98-4962-9924-097202093779.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195456Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=aa77b33e20fc1426307688f55147d57e8d3b2286a787c096b2da016d7d7d0c93","studio_link":"https://studio.reducto.ai/job/cddd635e-5d16-4621-bf48-031e7637dc0b","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16352969387356803,"top":0.10490237663507056,"width":0.11855640077405508,"height":0.011359045881442221,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7126715332269669},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:59 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:59.198829+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_legacy + custom_llm_provider: reducto + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: parse-legacy + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-legacy/7c4bbc65dfbc17bac7f0d179c5d7e201dfb9ac31de66094734ef38c288293625.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-legacy/7c4bbc65dfbc17bac7f0d179c5d7e201dfb9ac31de66094734ef38c288293625.yaml new file mode 100644 index 00000000000..deb47484ec8 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-legacy/7c4bbc65dfbc17bac7f0d179c5d7e201dfb9ac31de66094734ef38c288293625.yaml @@ -0,0 +1,39 @@ +interactions: +- request: + body: '{"document_url":"reducto://invalid-document-for-parity"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"error":{"code":404,"name":"NOT_FOUND","message":"Document ''The file + may have expired or been deleted. Please re-upload and try again.'' not found"},"detail":"Document + ''The file may have expired or been deleted. Please re-upload and try again.'' + not found"}' + headers: + Content-Type: + - application/json + Date: + - Wed, 02 Sep 2026 01:14:06 GMT + status: + code: 404 + message: '' +recorded_at: '2026-09-02T01:14:06.814847+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_legacy + document: + document_url: reducto://invalid-document-for-parity + type: document_url + model: reducto/parse-legacy + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-legacy/953d2a3bfea5594f9df19fc3f6c37e51b7e915b96a85e4ce8dfc41ba5f7f11df.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-legacy/953d2a3bfea5594f9df19fc3f6c37e51b7e915b96a85e4ce8dfc41ba5f7f11df.yaml new file mode 100644 index 00000000000..1e1c7c44463 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-legacy/953d2a3bfea5594f9df19fc3f6c37e51b7e915b96a85e4ce8dfc41ba5f7f11df.yaml @@ -0,0 +1,70 @@ +interactions: +- request: + body: "--90e8e7e6a71b2a4b14695234a736d695\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--90e8e7e6a71b2a4b14695234a736d695--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=90e8e7e6a71b2a4b14695234a736d695 + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://b9f242fd-fdb4-4b0a-9535-f11f432c7678.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:57 GMT + status: + code: 200 + message: '' +- request: + body: '{"document_url":"reducto://b9f242fd-fdb4-4b0a-9535-f11f432c7678.pdf","options":{"enhance":{}}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"25b78189-66fe-46d9-8114-14372a9d442d","duration":2.4146597385406494,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/a91f568b-6e98-4962-9924-097202093779.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195456Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=aa77b33e20fc1426307688f55147d57e8d3b2286a787c096b2da016d7d7d0c93","studio_link":"https://studio.reducto.ai/job/cddd635e-5d16-4621-bf48-031e7637dc0b","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16352969387356803,"top":0.10490237663507056,"width":0.11855640077405508,"height":0.011359045881442221,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7126715332269669},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:57 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:58.471820+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_legacy + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + enhance: {} + model: reducto/parse-legacy + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-legacy/a41d4c9547448df927b308d619d7e3b8fdbfe1fb5a9818a2fe47f02c2aad0fac.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-legacy/a41d4c9547448df927b308d619d7e3b8fdbfe1fb5a9818a2fe47f02c2aad0fac.yaml new file mode 100644 index 00000000000..a7a927cf93c --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-legacy/a41d4c9547448df927b308d619d7e3b8fdbfe1fb5a9818a2fe47f02c2aad0fac.yaml @@ -0,0 +1,69 @@ +interactions: +- request: + body: "--87aea09e72c511f78d32f6eabd194f5c\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--87aea09e72c511f78d32f6eabd194f5c--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=87aea09e72c511f78d32f6eabd194f5c + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://a91f568b-6e98-4962-9924-097202093779.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:53 GMT + status: + code: 200 + message: '' +- request: + body: '{"document_url":"reducto://a91f568b-6e98-4962-9924-097202093779.pdf"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"cddd635e-5d16-4621-bf48-031e7637dc0b","duration":2.4146597385406494,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/a91f568b-6e98-4962-9924-097202093779.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195456Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=aa77b33e20fc1426307688f55147d57e8d3b2286a787c096b2da016d7d7d0c93","studio_link":"https://studio.reducto.ai/job/cddd635e-5d16-4621-bf48-031e7637dc0b","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16352969387356803,"top":0.10490237663507056,"width":0.11855640077405508,"height":0.011359045881442221,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7126715332269669},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:57 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:57.246119+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_legacy + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: reducto/parse-legacy + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-v3/2148dfafc823aea933d7c9aa8877398c1c26cf1fb8f134b1326e79bd3d487c41.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-v3/2148dfafc823aea933d7c9aa8877398c1c26cf1fb8f134b1326e79bd3d487c41.yaml new file mode 100644 index 00000000000..474e6aba8c3 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-v3/2148dfafc823aea933d7c9aa8877398c1c26cf1fb8f134b1326e79bd3d487c41.yaml @@ -0,0 +1,72 @@ +interactions: +- request: + body: "--3c37fd2676c46ae1ff34a706baa70fad\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--3c37fd2676c46ae1ff34a706baa70fad--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=3c37fd2676c46ae1ff34a706baa70fad + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://671d6e00-6df5-493a-bd9e-8bbf3d77d3ab.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:36 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://671d6e00-6df5-493a-bd9e-8bbf3d77d3ab.pdf","retrieval":{"chunking":{"chunk_mode":"page"}}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"8f15b6dd-f2d2-48e2-a4c9-0230a9ace704","duration":3.7510643005371094,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/671d6e00-6df5-493a-bd9e-8bbf3d77d3ab.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195440Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=96a4f904e8d3ca58d399ab21ade69ec0ea62ff5dfee320c5eeaa6eaa0c4e0561","studio_link":"https://studio.reducto.ai/job/8f15b6dd-f2d2-48e2-a4c9-0230a9ace704","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:40 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:41.233002+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: reducto/parse-v3 + retrieval: + chunking: + chunk_mode: page + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-v3/50890748074024915ef138fcb8e0c95f731a0913d223fbe1780bbaaef1ecd835.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-v3/50890748074024915ef138fcb8e0c95f731a0913d223fbe1780bbaaef1ecd835.yaml new file mode 100644 index 00000000000..31f5ed2be1c --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-v3/50890748074024915ef138fcb8e0c95f731a0913d223fbe1780bbaaef1ecd835.yaml @@ -0,0 +1,83 @@ +interactions: +- request: + body: "--43159c32150fa5f506932c4b4a645ef6\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--43159c32150fa5f506932c4b4a645ef6--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=43159c32150fa5f506932c4b4a645ef6 + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://f8bf17e8-ca0b-4add-b484-9982d2e4ac2a.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:50 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://f8bf17e8-ca0b-4add-b484-9982d2e4ac2a.pdf","formatting":{"add_page_markers":true,"table_output_format":"json","merge_tables":true,"include":["change_tracking","highlight","comments"]}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"e3792387-8384-45aa-b02e-a84521db116a","duration":1.0497362613677979,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/f8bf17e8-ca0b-4add-b484-9982d2e4ac2a.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195452Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=7367275a69dfd53f9fcf9f02777f0fe973e7e541fa1871e0347f788b8c5078f4","studio_link":"https://studio.reducto.ai/job/e3792387-8384-45aa-b02e-a84521db116a","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"[[START + OF PAGE 1]]\n\n# Test PDF File\n\n[[END OF PAGE 1]]","embed":"[[START OF PAGE + 1]]\n\n# Test PDF File\n\n[[END OF PAGE 1]]","enriched":null,"enrichment_success":false,"blocks":[{"type":"Page + Number","bbox":{"left":0.0,"top":0.0,"width":0.0,"height":0.0,"page":1,"original_page":1},"content":"[[START + OF PAGE 1]]","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":null},"extra":null},{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null},{"type":"Page + Number","bbox":{"left":0.0,"top":0.0,"width":0.0,"height":0.0,"page":1,"original_page":1},"content":"[[END + OF PAGE 1]]","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":null},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:52 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:53.454592+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + custom_llm_provider: null + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + formatting: + add_page_markers: true + include: + - change_tracking + - highlight + - comments + merge_tables: true + table_output_format: json + model: reducto/parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-v3/7cefa9e08179db20d45e255d269e4239015b9b44376ce49c921f8fc057fb9c6a.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-v3/7cefa9e08179db20d45e255d269e4239015b9b44376ce49c921f8fc057fb9c6a.yaml new file mode 100644 index 00000000000..02bd747a4d3 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-v3/7cefa9e08179db20d45e255d269e4239015b9b44376ce49c921f8fc057fb9c6a.yaml @@ -0,0 +1,70 @@ +interactions: +- request: + body: "--ba392e7bb7694af286b0acfe46365a0f\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--ba392e7bb7694af286b0acfe46365a0f--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=ba392e7bb7694af286b0acfe46365a0f + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://a6d3c3bb-a6f1-4636-8b3a-ee920b29d387.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:46 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://a6d3c3bb-a6f1-4636-8b3a-ee920b29d387.pdf"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"4a78c975-1865-46fe-8f89-bea92a49e215","duration":1.1194427013397217,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/6ab9bcf4-1893-4b37-bddb-acda8ce45dfb.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195445Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=1db61d33f602f13e8444e8580b737b31f346e47493cc7eca7c4fcddbde0dfb43","studio_link":"https://studio.reducto.ai/job/d345549f-5d98-4c62-b3e3-24c990144df4","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:47 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:47.967951+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + custom_llm_provider: reducto + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-v3/8270ee13cdfb2ceefd8302a1880653fe30bde512da747d2fd8e78759ccdc9b90.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-v3/8270ee13cdfb2ceefd8302a1880653fe30bde512da747d2fd8e78759ccdc9b90.yaml new file mode 100644 index 00000000000..2bc911985eb --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-v3/8270ee13cdfb2ceefd8302a1880653fe30bde512da747d2fd8e78759ccdc9b90.yaml @@ -0,0 +1,70 @@ +interactions: +- request: + body: "--0f6127f8c781afd158616115a5ebdece\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--0f6127f8c781afd158616115a5ebdece--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=0f6127f8c781afd158616115a5ebdece + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://6ab9bcf4-1893-4b37-bddb-acda8ce45dfb.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:44 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://6ab9bcf4-1893-4b37-bddb-acda8ce45dfb.pdf"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"d345549f-5d98-4c62-b3e3-24c990144df4","duration":1.1194427013397217,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/6ab9bcf4-1893-4b37-bddb-acda8ce45dfb.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195445Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=1db61d33f602f13e8444e8580b737b31f346e47493cc7eca7c4fcddbde0dfb43","studio_link":"https://studio.reducto.ai/job/d345549f-5d98-4c62-b3e3-24c990144df4","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:46 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:46.694836+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + custom_llm_provider: null + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: reducto/parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-v3/997cd381eb3019b71086e4fdab1b98a3a0d251a27c3bb80178580a711c111e94.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-v3/997cd381eb3019b71086e4fdab1b98a3a0d251a27c3bb80178580a711c111e94.yaml new file mode 100644 index 00000000000..622db8dee4a --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-v3/997cd381eb3019b71086e4fdab1b98a3a0d251a27c3bb80178580a711c111e94.yaml @@ -0,0 +1,100 @@ +interactions: +- request: + body: "--22b7cb49f85a1ae5113a6328b0381ffb\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--22b7cb49f85a1ae5113a6328b0381ffb--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=22b7cb49f85a1ae5113a6328b0381ffb + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://5df745f4-2877-4cf6-9bde-a2f829c93eea.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:48 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://5df745f4-2877-4cf6-9bde-a2f829c93eea.pdf","formatting":{"add_page_markers":false,"table_output_format":"json","merge_tables":true,"include":["signatures","ignore_watermarks"]},"retrieval":{"chunking":{"chunk_mode":"variable","chunk_size":1500,"chunk_overlap":32},"filter_blocks":["Figure","Table","Key + Value"],"embedding_optimized":false},"settings":{"ocr_system":"legacy","extraction_mode":"hybrid","force_url_result":false,"return_ocr_data":false,"return_images":[],"embed_pdf_metadata":false,"embed_pdf_metadata_dpi":100,"persist_results":false,"timeout":900.0,"page_range":[1]}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"7ca67242-1677-484e-b2ef-b256dcdd1ea4","duration":1.3359308242797852,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/5df745f4-2877-4cf6-9bde-a2f829c93eea.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195449Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=077a6551201d2562fd1543fd5c391541cffe606d64307a3f6d103cf18219f80a","studio_link":"https://studio.reducto.ai/job/7ca67242-1677-484e-b2ef-b256dcdd1ea4","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:50 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:50.708637+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + custom_llm_provider: null + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + formatting: + add_page_markers: false + include: + - signatures + - ignore_watermarks + merge_tables: true + table_output_format: json + model: reducto/parse-v3 + retrieval: + chunking: + chunk_mode: variable + chunk_overlap: 32 + chunk_size: 1500 + embedding_optimized: false + filter_blocks: + - Figure + - Table + - Key Value + settings: + embed_pdf_metadata: false + embed_pdf_metadata_dpi: 100 + extraction_mode: hybrid + force_url_result: false + ocr_system: legacy + page_range: + - 1 + persist_results: false + return_images: [] + return_ocr_data: false + timeout: 900.0 + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-v3/f2b260eca76347d5982b96c79563999653d141626ca1eadfdb47fc20d645b9d5.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-v3/f2b260eca76347d5982b96c79563999653d141626ca1eadfdb47fc20d645b9d5.yaml new file mode 100644 index 00000000000..4515e0a0823 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-v3/f2b260eca76347d5982b96c79563999653d141626ca1eadfdb47fc20d645b9d5.yaml @@ -0,0 +1,72 @@ +interactions: +- request: + body: "--7108360c769817ee2464c4729615e289\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--7108360c769817ee2464c4729615e289--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=7108360c769817ee2464c4729615e289 + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://fc6ebcb1-95d4-46ec-90d0-efca33ef197f.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:41 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://fc6ebcb1-95d4-46ec-90d0-efca33ef197f.pdf","settings":{"return_ocr_data":true}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"3f6ef64d-c949-4b8d-9898-afb3b01669e6","duration":1.4480743408203125,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/fc6ebcb1-95d4-46ec-90d0-efca33ef197f.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195443Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=0965ca129cee49a41650df152057a17a4f9e265f92f2261a4f6576587272eb03","studio_link":"https://studio.reducto.ai/job/3f6ef64d-c949-4b8d-9898-afb3b01669e6","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":{"words":[{"text":"Test","bbox":{"left":0.16189475464665032,"top":0.09491921434498797,"width":0.03832089043910207,"height":0.021019531018806225,"page":1,"original_page":1},"confidence":1.0,"chunk_index":null,"rotation":359},{"text":"PDF","bbox":{"left":0.20548195932425706,"top":0.09514990719881924,"width":0.03939929039649714,"height":0.02102524343163076,"page":1,"original_page":1},"confidence":1.0,"chunk_index":null,"rotation":359},{"text":"File","bbox":{"left":0.25014757642558977,"top":0.09538630283240115,"width":0.03177201514150582,"height":0.020984871218902895,"page":1,"original_page":1},"confidence":1.0,"chunk_index":null,"rotation":359}],"lines":[{"text":"Test + PDF File","bbox":{"left":0.16189475464665032,"top":0.09491921434498797,"width":0.12002483692044526,"height":0.021451959706316092,"page":1,"original_page":1},"confidence":1.0,"chunk_index":null,"rotation":359}]},"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:43 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:43.968997+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: reducto/parse-v3 + settings: + return_ocr_data: true + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-v3/f317c1c7b57a6ac742de1b97f2f84da6aa7892f8cdd4f22158ba4c700dbe8a81.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-v3/f317c1c7b57a6ac742de1b97f2f84da6aa7892f8cdd4f22158ba4c700dbe8a81.yaml new file mode 100644 index 00000000000..3349f3f6b2d --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-v3/f317c1c7b57a6ac742de1b97f2f84da6aa7892f8cdd4f22158ba4c700dbe8a81.yaml @@ -0,0 +1,39 @@ +interactions: +- request: + body: '{"input":"reducto://invalid-document-for-parity"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"error":{"code":404,"name":"NOT_FOUND","message":"Document ''The file + may have expired or been deleted. Please re-upload and try again.'' not found"},"detail":"Document + ''The file may have expired or been deleted. Please re-upload and try again.'' + not found"}' + headers: + Content-Type: + - application/json + Date: + - Wed, 02 Sep 2026 01:14:02 GMT + status: + code: 404 + message: '' +recorded_at: '2026-09-02T01:14:02.538070+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + document: + document_url: reducto://invalid-document-for-parity + type: document_url + model: reducto/parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-v3/f55e3b684697b9f478393e35202e92b04e51b1ea8b74eaa4a4382a01154b5f5a.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-v3/f55e3b684697b9f478393e35202e92b04e51b1ea8b74eaa4a4382a01154b5f5a.yaml new file mode 100644 index 00000000000..5731fa1c48d --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-v3/f55e3b684697b9f478393e35202e92b04e51b1ea8b74eaa4a4382a01154b5f5a.yaml @@ -0,0 +1,71 @@ +interactions: +- request: + body: "--04abc28545d4b46a8ee703e56d88cc0c\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--04abc28545d4b46a8ee703e56d88cc0c--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=04abc28545d4b46a8ee703e56d88cc0c + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://f3cc3c72-614a-4104-bb70-a516f37148e0.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:30 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://f3cc3c72-614a-4104-bb70-a516f37148e0.pdf","formatting":{"table_output_format":"md"}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"aba924dc-7773-4dff-a4a8-3549366ea9fa","duration":4.5764172077178955,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/f3cc3c72-614a-4104-bb70-a516f37148e0.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195435Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=7d4c9464bb3a1a9d5c7a5f957bc10813fb16fcc6de3fea2054a40b469ffa79b3","studio_link":"https://studio.reducto.ai/job/aba924dc-7773-4dff-a4a8-3549366ea9fa","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:35 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:35.993066+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + formatting: + table_output_format: md + model: reducto/parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/data/reducto-v3/f847dae1a1f6c3777ce3b65421a97f703d67d234593ac188f948142190fdf06d.yaml b/tests/test_litellm/ocr/fixtures/data/reducto-v3/f847dae1a1f6c3777ce3b65421a97f703d67d234593ac188f948142190fdf06d.yaml new file mode 100644 index 00000000000..02298f5a2f0 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/data/reducto-v3/f847dae1a1f6c3777ce3b65421a97f703d67d234593ac188f948142190fdf06d.yaml @@ -0,0 +1,69 @@ +interactions: +- request: + body: "--b7daa0c5be8f80d33a4eb7a54318d93a\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--b7daa0c5be8f80d33a4eb7a54318d93a--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=b7daa0c5be8f80d33a4eb7a54318d93a + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://010b01b2-83bd-446d-af11-d120c5ac2c02.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:27 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://010b01b2-83bd-446d-af11-d120c5ac2c02.pdf"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"8138a6c6-b726-4300-90b4-e6d5d43f6f70","duration":1.2648272514343262,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/010b01b2-83bd-446d-af11-d120c5ac2c02.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195428Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=3cda9020fefff85d9dd13f124908d7b00e55b6e82a72039546d262099cfcf3b4","studio_link":"https://studio.reducto.ai/job/8138a6c6-b726-4300-90b4-e6d5d43f6f70","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:29 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:30.255600+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: reducto/parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/test_litellm/ocr/fixtures/migrate.py b/tests/test_litellm/ocr/fixtures/migrate.py new file mode 100644 index 00000000000..41265896888 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/migrate.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Final, cast + +import litellm +from litellm.rust_bridge.ocr import use_litellm_rust +from tests.route_parity.fixtures.recording import ( + RecordedInteraction, + UpstreamEndpoint, + record_upstream_interactions, +) +from tests.route_parity.fixtures.store import FixtureEnvelope, read_fixture, save_fixture +from tests.route_parity.replay import replay_server +from tests.test_litellm.ocr.fixtures.common import OcrSdkCall +from tests.test_litellm.ocr.fixtures.config import configured_fixture_directory +from tests.test_litellm.ocr.fixtures.models import OcrParityCase, OcrSdkInput + + +def _invoke(provider_url: str, case_input: OcrSdkInput) -> object: + sdk_call: Final = cast(OcrSdkCall, litellm.ocr) + return sdk_call(api_base=provider_url, api_key="test-key", **case_input.as_sdk_kwargs()) + + +def migrate_fixture(path: Path) -> Path: + case: Final = read_fixture(path, OcrParityCase) + envelope: Final = FixtureEnvelope.model_validate_json(path.read_text(encoding="utf-8")) + with replay_server() as provider: + for response in case.provider_responses: + provider.enqueue_response(response) + captured: Final = record_upstream_interactions(UpstreamEndpoint(provider.url), case.litellm_input, _invoke) + provider.take_requests(len(case.provider_responses)) + interactions: Final = tuple( + RecordedInteraction(item.request, response) + for item, response in zip(captured, case.provider_responses, strict=True) + ) + destination: Final = save_fixture( + path.parent, + case.litellm_input, + case, + interactions, + recorded_at=envelope.recorded_at, + request_source="python_replay", + ) + read_fixture(destination, OcrParityCase) + path.unlink() + return destination + + +def main() -> None: + parser: Final = argparse.ArgumentParser() + parser.add_argument("--fixture-dir", type=Path, default=configured_fixture_directory()) + args: Final = parser.parse_args() + directory: Final = cast(Path, args.fixture_dir) + use_litellm_rust(False, ocr=None, aocr=None) + paths: Final = tuple(sorted(directory.rglob("*.json"))) + for path in paths: + print(f"Migrated {path.name} to {migrate_fixture(path).name}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_litellm/ocr/fixtures/mistral.py b/tests/test_litellm/ocr/fixtures/mistral.py new file mode 100644 index 00000000000..52adf113b91 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral.py @@ -0,0 +1,259 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final, Literal, cast + +from hypothesis import strategies as st +from hypothesis.strategies import SearchStrategy +from pydantic import model_validator +from typing_extensions import Self + +from tests.route_parity.fixtures.recording import UpstreamEndpoint +from tests.test_litellm.ocr.fixtures.base import ( + JsonSchemaResponseFormat, + OcrDocument, + OcrSdkInputBase, +) +from tests.test_litellm.ocr.fixtures.common import ( + OcrFixtureClient, + OcrRecordingTarget, + annotation_format, + document_transport_strategy, + invoke_with_api_key, + pdf_document, +) + +MistralModel = Literal[ + "mistral/mistral-ocr-3", + "mistral/mistral-ocr-3-0", + "mistral/mistral-ocr-2512", + "mistral/mistral-ocr-4-0", + "mistral/mistral-ocr-4-1", + "mistral/mistral-ocr-4", + "mistral/mistral-ocr-latest", + "mistral-ocr-3", + "mistral-ocr-3-0", + "mistral-ocr-2512", + "mistral-ocr-4-0", + "mistral-ocr-4-1", + "mistral-ocr-4", + "mistral-ocr-latest", +] +MistralFixtureModel = MistralModel | Literal["mistral/invalid-ocr-model-for-parity"] + +MISTRAL_MODELS: Final[tuple[MistralModel, ...]] = ( + "mistral/mistral-ocr-3", + "mistral/mistral-ocr-3-0", + "mistral/mistral-ocr-2512", + "mistral/mistral-ocr-4", + "mistral/mistral-ocr-4-0", + "mistral/mistral-ocr-4-1", + "mistral/mistral-ocr-latest", +) + + +class MistralCompatibleOcrSdkInput(OcrSdkInputBase): + document: OcrDocument + pages: str | list[int] | None = None + include_image_base64: bool | None = None + image_limit: int | None = None + image_min_size: int | None = None + bbox_annotation_format: JsonSchemaResponseFormat | None = None + document_annotation_format: JsonSchemaResponseFormat | None = None + document_annotation_prompt: str | None = None + extract_header: bool = False + extract_footer: bool = False + table_format: Literal["markdown", "html"] | None = None + confidence_scores_granularity: Literal["page", "word", "block"] | None = None + include_blocks: bool = True + id: str | None = None + + @model_validator(mode="after") + def validate_annotation_prompt(self) -> Self: + if self.document_annotation_prompt is not None and self.document_annotation_format is None: + raise ValueError("document_annotation_prompt requires document_annotation_format") + return self + + +class MistralOcrSdkInput(MistralCompatibleOcrSdkInput): + contract: Literal["mistral"] = "mistral" + model: MistralFixtureModel + custom_llm_provider: Literal["mistral"] | None = None + + @model_validator(mode="after") + def validate_provider_routing(self) -> Self: + if not self.model.startswith("mistral/") and self.custom_llm_provider != "mistral": + raise ValueError("unqualified Mistral models require custom_llm_provider='mistral'") + return self + + +MISTRAL_MODEL: Final[MistralModel] = "mistral/mistral-ocr-latest" +MISTRAL_PROVIDER_REJECTED_INPUTS: Final[tuple[MistralOcrSdkInput, ...]] = ( + MistralOcrSdkInput( + model="mistral/invalid-ocr-model-for-parity", + document=pdf_document(), + ), +) +MistralFeatureLevel = Literal["2505", "2512", "4"] +_MISTRAL_4_MODELS: Final = frozenset( + { + "mistral/mistral-ocr-4", + "mistral/mistral-ocr-4-0", + "mistral/mistral-ocr-4-1", + "mistral/mistral-ocr-latest", + } +) +_MISTRAL_2512_MODELS: Final = frozenset( + {*_MISTRAL_4_MODELS, "mistral/mistral-ocr-2512", "mistral/mistral-ocr-3", "mistral/mistral-ocr-3-0"} +) + + +def _feature_level(model: str) -> MistralFeatureLevel: + if model in _MISTRAL_4_MODELS: + return "4" + if model in _MISTRAL_2512_MODELS: + return "2512" + return "2505" + + +def _optional_param_strategies( + *, + include_document_annotation_prompt: bool = True, +) -> tuple[ + tuple[SearchStrategy[dict[str, object]], ...], + tuple[SearchStrategy[dict[str, object]], ...], + tuple[SearchStrategy[dict[str, object]], ...], +]: + annotation: Final = annotation_format("document_title") + common: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( + st.sampled_from(((0,), (0, 1))).map(list).map(lambda value: {"pages": value}), + st.sampled_from((False, True)).map(lambda value: {"include_image_base64": value}), + st.just({"image_limit": 1}), + st.just({"image_min_size": 300}), + st.just({"bbox_annotation_format": annotation_format("bounding_boxes")}), + st.just({"document_annotation_format": annotation}), + *( + ( + st.just( + { + "document_annotation_format": annotation, + "document_annotation_prompt": "Extract the visible title", + } + ), + ) + if include_document_annotation_prompt + else () + ), + st.sampled_from(("page", "word")).map(lambda value: {"confidence_scores_granularity": value}), + ) + feature_2512: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( + st.sampled_from((False, True)).map(lambda value: {"extract_header": value}), + st.sampled_from((False, True)).map(lambda value: {"extract_footer": value}), + st.sampled_from(("markdown", "html")).map(lambda value: {"table_format": value}), + ) + feature_4: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( + st.just({"pages": "0-2"}), + st.sampled_from((False, True)).map(lambda value: {"include_blocks": value}), + st.just({"include_blocks": True, "confidence_scores_granularity": "block"}), + ) + return common, feature_2512, feature_4 + + +def mistral_optional_params_strategy( + feature_level: MistralFeatureLevel, + *, + include_document_annotation_prompt: bool = True, +) -> SearchStrategy[dict[str, object]]: + common, feature_2512, feature_4 = _optional_param_strategies( + include_document_annotation_prompt=include_document_annotation_prompt + ) + return st.one_of( + *common, + *(feature_2512 if feature_level in {"2512", "4"} else ()), + *(feature_4 if feature_level == "4" else ()), + ) + + +def _mistral_input_values( + document: OcrDocument, + optional_params: dict[str, object] | None = None, +) -> dict[str, object]: + return {"document": document, **(optional_params or {})} + + +def _mistral_input( + model: str, + document: OcrDocument, + optional_params: dict[str, object] | None = None, +) -> MistralOcrSdkInput: + return MistralOcrSdkInput.model_validate({"model": model, **_mistral_input_values(document, optional_params)}) + + +def mistral_input_values_strategy( + feature_level: MistralFeatureLevel, + inline_image_data_uri: str, + *, + include_document_annotation_prompt: bool = True, +) -> SearchStrategy[dict[str, object]]: + option_document: Final = pdf_document() + return st.one_of( + document_transport_strategy(inline_image_data_uri).map(_mistral_input_values), + mistral_optional_params_strategy( + feature_level, + include_document_annotation_prompt=include_document_annotation_prompt, + ).map(lambda optional_params: _mistral_input_values(option_document, optional_params)), + ) + + +def mistral_input_strategy( + model: str, + inline_image_data_uri: str, + feature_level: MistralFeatureLevel | None = None, +) -> SearchStrategy[MistralOcrSdkInput]: + return mistral_input_values_strategy(feature_level or _feature_level(model), inline_image_data_uri).map( + lambda values: MistralOcrSdkInput.model_validate({"model": model, **values}) + ) + + +def _mistral_recording_strategy(inline_image_data_uri: str) -> SearchStrategy[MistralOcrSdkInput]: + document: Final = pdf_document() + baseline_models: Final = tuple(model for model in MISTRAL_MODELS if model != MISTRAL_MODEL) + common, feature_2512, feature_4 = _optional_param_strategies() + common_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*common) + feature_2512_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*feature_2512) + feature_4_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*feature_4) + return st.one_of( + st.sampled_from(baseline_models).map(lambda model: _mistral_input(model, document)), + document_transport_strategy(inline_image_data_uri).map( + lambda selected_document: _mistral_input(MISTRAL_MODEL, selected_document) + ), + common_options.map(lambda optional_params: _mistral_input(MISTRAL_MODEL, document, optional_params)), + feature_2512_options.map( + lambda optional_params: _mistral_input("mistral/mistral-ocr-2512", document, optional_params) + ), + feature_4_options.map( + lambda optional_params: _mistral_input("mistral/mistral-ocr-4-1", document, optional_params) + ), + ) + + +def mistral_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str +) -> tuple[OcrRecordingTarget, ...]: + api_key: Final = environ.get("MISTRAL_API_KEY") + if not api_key: + return () + configured: Final = environ.get("MISTRAL_API_BASE", "https://api.mistral.ai").rstrip("/") + base_url: Final = configured.removesuffix("/v1") + return ( + OcrRecordingTarget( + name="mistral-ocr", + upstream=UpstreamEndpoint(base_url=base_url), + strategy=cast( + SearchStrategy[OcrSdkInputBase], + _mistral_recording_strategy(inline_image_data_uri), + ), + invocation=invoke_with_api_key(client, api_key), + required_inputs=MISTRAL_PROVIDER_REJECTED_INPUTS, + ), + ) diff --git a/tests/test_litellm/ocr/fixtures/models.py b/tests/test_litellm/ocr/fixtures/models.py new file mode 100644 index 00000000000..0dd8b7399df --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/models.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Annotated, Final, cast + +from pydantic import Field, model_validator + +from tests.route_parity.fixture_models import ParityCase +from tests.test_litellm.ocr.fixtures.azure import ( + AzureDocumentIntelligenceOcrSdkInput, + AzureMistralOcrSdkInput, +) +from tests.test_litellm.ocr.fixtures.mistral import MistralOcrSdkInput +from tests.test_litellm.ocr.fixtures.reducto import ReductoParseLegacySdkInput, ReductoParseV3SdkInput +from tests.test_litellm.ocr.fixtures.vertex import VertexDeepSeekOcrSdkInput, VertexMistralOcrSdkInput + +__all__ = ("OcrParityCase", "OcrSdkInput") + + +OcrSdkInput = Annotated[ + MistralOcrSdkInput + | AzureMistralOcrSdkInput + | VertexMistralOcrSdkInput + | AzureDocumentIntelligenceOcrSdkInput + | VertexDeepSeekOcrSdkInput + | ReductoParseV3SdkInput + | ReductoParseLegacySdkInput, + Field(discriminator="contract"), +] + + +class OcrParityCase(ParityCase[OcrSdkInput]): + @model_validator(mode="before") + @classmethod + def load_legacy_contract(cls, value: object) -> object: + if not isinstance(value, Mapping): + return value + fixture: Final = cast(Mapping[str, object], value) + litellm_input: Final = fixture.get("litellm_input") + if not isinstance(litellm_input, Mapping) or "contract" in litellm_input: + return fixture + legacy_input: Final = cast(Mapping[str, object], litellm_input) + legacy_contract: Final = legacy_input.get("boundary") + if isinstance(legacy_contract, str): + return { + **fixture, + "litellm_input": { + "contract": legacy_contract, + **{key: item for key, item in legacy_input.items() if key != "boundary"}, + }, + } + model: Final = legacy_input.get("model") + if not isinstance(model, str): + return fixture + return {**fixture, "litellm_input": {"contract": _legacy_contract(model), **legacy_input}} + + +def _legacy_contract(model: str) -> str: + if model.startswith("azure_ai/doc-intelligence/"): + return "azure_document_intelligence" + if model.startswith("azure_ai/"): + return "azure_mistral" + if model.startswith("vertex_ai/deepseek"): + return "vertex_deepseek" + if model.startswith("vertex_ai/"): + return "vertex_mistral" + if model.endswith("parse-v3"): + return "reducto_v3" + if model.endswith("parse-legacy"): + return "reducto_legacy" + return "mistral" diff --git a/tests/test_litellm/ocr/fixtures/record.py b/tests/test_litellm/ocr/fixtures/record.py new file mode 100644 index 00000000000..72fcc2c8e20 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/record.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import logging +import os +from collections.abc import Mapping +from typing import Final, cast + +from dotenv import load_dotenv + +import litellm +from litellm.rust_bridge.ocr import use_litellm_rust +from tests.route_parity.fixtures.cli import parse_recording_args +from tests.route_parity.fixtures.media import structured_image_data_uri +from tests.route_parity.fixtures.pipeline import record_fixtures +from tests.route_parity.fixtures.store import fixture_directory +from tests.test_litellm.ocr.fixtures.azure import ( + azure_document_intelligence_recording_targets, + azure_mistral_recording_targets, +) +from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase +from tests.test_litellm.ocr.fixtures.common import OcrFixtureClient, OcrRecordingTarget, OcrSdkCall +from tests.test_litellm.ocr.fixtures.config import DEFAULT_FIXTURE_DIRECTORY, FIXTURE_DIR_ENV +from tests.test_litellm.ocr.fixtures.mistral import mistral_recording_targets +from tests.test_litellm.ocr.fixtures.models import OcrParityCase +from tests.test_litellm.ocr.fixtures.reducto import reducto_recording_targets +from tests.test_litellm.ocr.fixtures.vertex import vertex_recording_targets + + +class LiteLLMOcrFixtureClient: + def __init__(self, sdk_call: OcrSdkCall) -> None: + self.sdk_call: Final = sdk_call + + def execute(self, api_base: str, api_key: str, case_input: OcrSdkInputBase) -> None: + self.sdk_call(api_base=api_base, api_key=api_key, **case_input.as_sdk_kwargs()) + + +def discover_targets( + environ: Mapping[str, str], + client: OcrFixtureClient, + inline_image_data_uri: str, +) -> tuple[OcrRecordingTarget, ...]: + return ( + *mistral_recording_targets(environ, client, inline_image_data_uri), + *azure_mistral_recording_targets(environ, client, inline_image_data_uri), + *azure_document_intelligence_recording_targets(environ, client), + *vertex_recording_targets(environ, client, inline_image_data_uri), + *reducto_recording_targets(environ, client, inline_image_data_uri), + ) + + +def require_targets(targets: tuple[OcrRecordingTarget, ...]) -> tuple[OcrRecordingTarget, ...]: + if targets: + return targets + raise SystemExit("No OCR fixture providers are configured. Set a supported provider API key and endpoint") + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(message)s") + load_dotenv() + args: Final = parse_recording_args() + client: Final = LiteLLMOcrFixtureClient(cast(OcrSdkCall, litellm.ocr)) + inline_image_data_uri: Final = structured_image_data_uri() + targets: Final = require_targets(discover_targets(os.environ, client, inline_image_data_uri)) + root: Final = fixture_directory( + args.fixture_dir, + os.environ.get(FIXTURE_DIR_ENV), + DEFAULT_FIXTURE_DIRECTORY, + ) + use_litellm_rust(False, ocr=None, aocr=None) + summary: Final = record_fixtures(targets, root, args.examples, args.concurrency, OcrParityCase) + return summary.exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_litellm/ocr/fixtures/reducto.py b/tests/test_litellm/ocr/fixtures/reducto.py new file mode 100644 index 00000000000..59af22694bf --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +import base64 +import binascii +from collections.abc import Mapping +from typing import Annotated, Final, Literal, cast + +from hypothesis import strategies as st +from hypothesis.strategies import SearchStrategy +from pydantic import Field, field_validator, model_validator +from typing_extensions import Self + +from tests.route_parity.fixture_models import FixtureModel, JsonObject +from tests.route_parity.fixtures.media import structured_pdf_data_uri +from tests.route_parity.fixtures.recording import UpstreamEndpoint +from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase +from tests.test_litellm.ocr.fixtures.common import ( + OcrFixtureClient, + OcrRecordingTarget, + image_data_document, + invoke_with_api_key, +) + + +def _validate_reducto_source(source: str) -> str: + if source.startswith("reducto://"): + return source + if not source.startswith("data:"): + raise ValueError("Reducto documents require a reducto:// id or base64 data URI") + try: + header, encoded = source.split(",", 1) + except ValueError as error: + raise ValueError("invalid Reducto data URI") from error + if ";base64" not in header: + raise ValueError("Reducto data URIs must be base64 encoded") + try: + base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError) as error: + raise ValueError("invalid Reducto base64 payload") from error + return source + + +class ReductoImageUrlDocument(FixtureModel): + type: Literal["image_url"] + image_url: str + + @field_validator("image_url") + @classmethod + def validate_image_url(cls, value: str) -> str: + return _validate_reducto_source(value) + + +class ReductoDocumentUrlDocument(FixtureModel): + type: Literal["document_url"] + document_url: str + + @field_validator("document_url") + @classmethod + def validate_document_url(cls, value: str) -> str: + return _validate_reducto_source(value) + + +ReductoDocument = Annotated[ + ReductoImageUrlDocument | ReductoDocumentUrlDocument, + Field(discriminator="type"), +] + +ReductoTableOutputFormat = Literal["html", "json", "md", "jsonbbox", "dynamic", "csv"] +ReductoReturnImage = Literal["figure", "table", "page"] +ReductoFormattingInclude = Literal[ + "change_tracking", + "highlight", + "comments", + "hyperlinks", + "signatures", + "ignore_watermarks", +] +ReductoBlockType = Literal[ + "Header", + "Footer", + "Title", + "Section Header", + "Page Number", + "List Item", + "Figure", + "Table", + "Key Value", + "Text", + "Comment", + "Signature", +] +_REDUCTO_FILTER_BLOCK_GROUPS: Final[tuple[tuple[ReductoBlockType, ...], ...]] = ( + (), + ("Header",), + ("Header", "Footer", "Page Number"), + ("Figure", "Table", "Key Value"), +) +_REDUCTO_RETURN_IMAGE_GROUPS: Final[tuple[tuple[ReductoReturnImage, ...], ...]] = ( + (), + ("figure",), + ("table",), + ("page",), + ("figure", "table"), +) + + +class ReductoFormatting(FixtureModel): + add_page_markers: bool = False + table_output_format: ReductoTableOutputFormat = "dynamic" + merge_tables: bool = False + include: list[ReductoFormattingInclude] = Field(default_factory=list) + + @field_validator("include") + @classmethod + def validate_unique_include(cls, value: list[ReductoFormattingInclude]) -> list[ReductoFormattingInclude]: + if len(value) != len(set(value)): + raise ValueError("formatting.include entries must be unique") + return value + + +class ReductoChunking(FixtureModel): + chunk_mode: Literal["variable", "section", "page", "disabled", "block", "page_sections"] = "disabled" + chunk_size: int | None = None + chunk_overlap: int = Field(default=0, ge=0) + + @model_validator(mode="after") + def validate_chunking(self) -> Self: + if self.chunk_size is not None and self.chunk_size <= 0: + raise ValueError("chunk_size must be positive") + if self.chunk_size is not None and self.chunk_overlap >= self.chunk_size: + raise ValueError("chunk_overlap must be less than chunk_size") + return self + + +class ReductoRetrieval(FixtureModel): + chunking: ReductoChunking = Field(default_factory=ReductoChunking) + filter_blocks: list[ReductoBlockType] = Field(default_factory=list) + embedding_optimized: bool = False + + @field_validator("filter_blocks") + @classmethod + def validate_unique_blocks(cls, value: list[ReductoBlockType]) -> list[ReductoBlockType]: + if len(value) != len(set(value)): + raise ValueError("retrieval.filter_blocks entries must be unique") + return value + + +class ReductoPageRange(FixtureModel): + start: int | None = Field(default=None, ge=1) + end: int | None = Field(default=None, ge=1) + + @model_validator(mode="after") + def validate_range(self) -> Self: + if self.start is not None and self.end is not None and self.end < self.start: + raise ValueError("page range end must be greater than or equal to start") + return self + + +class ReductoTenantThrottling(FixtureModel): + tenant_id: str = Field(min_length=1, max_length=256) + max_share: float = Field(default=0.5, gt=0, le=1) + + +class ReductoHybridVpcSettings(FixtureModel): + environment: str | None = None + + +ReductoPageSelection = ReductoPageRange | list[ReductoPageRange] | list[int] | list[str] +ReductoV3Model = Literal["reducto/parse-v3", "parse-v3"] +ReductoLegacyModel = Literal["reducto/parse-legacy", "parse-legacy"] +_ReductoV3Route = Literal["qualified", "image", "unqualified"] +_ReductoLegacyRoute = Literal["qualified", "unqualified"] + +REDUCTO_V3_MODELS: Final[tuple[Literal["reducto/parse-v3"], ...]] = ("reducto/parse-v3",) +REDUCTO_LEGACY_MODELS: Final[tuple[Literal["reducto/parse-legacy"], ...]] = ("reducto/parse-legacy",) + + +class ReductoSettings(FixtureModel): + model: Literal["r-1"] | None = None + ocr_system: Literal["standard", "legacy"] = "standard" + extraction_mode: Literal["ocr", "hybrid", "metadata"] = "hybrid" + force_url_result: bool = False + force_file_extension: str | None = None + return_ocr_data: bool = False + return_images: list[ReductoReturnImage] = Field(default_factory=list) + embed_pdf_metadata: bool = False + embed_pdf_metadata_dpi: int = Field(default=100, ge=50, le=250) + persist_results: bool = False + tenant_throttling: ReductoTenantThrottling | None = None + timeout: float | None = Field(default=None, gt=0) + page_range: ReductoPageSelection | None = None + document_password: str | None = None + hybrid_vpc: ReductoHybridVpcSettings = Field(default_factory=ReductoHybridVpcSettings) + + @field_validator("return_images") + @classmethod + def validate_unique_images(cls, value: list[ReductoReturnImage]) -> list[ReductoReturnImage]: + if len(value) != len(set(value)): + raise ValueError("settings.return_images entries must be unique") + return value + + +class ReductoParseV3SdkInput(OcrSdkInputBase): + contract: Literal["reducto_v3"] = "reducto_v3" + model: ReductoV3Model + document: ReductoDocument + custom_llm_provider: Literal["reducto"] | None = None + formatting: ReductoFormatting = Field(default_factory=ReductoFormatting) + retrieval: ReductoRetrieval = Field(default_factory=ReductoRetrieval) + settings: ReductoSettings = Field(default_factory=ReductoSettings) + + @model_validator(mode="after") + def validate_provider_routing(self) -> Self: + if self.model == "parse-v3" and self.custom_llm_provider != "reducto": + raise ValueError("unqualified Reducto models require custom_llm_provider='reducto'") + return self + + +class ReductoParseLegacySdkInput(OcrSdkInputBase): + contract: Literal["reducto_legacy"] = "reducto_legacy" + model: ReductoLegacyModel + document: ReductoDocument + custom_llm_provider: Literal["reducto"] | None = None + enhance: JsonObject | None = None + + @model_validator(mode="after") + def validate_provider_routing(self) -> Self: + if self.model == "parse-legacy" and self.custom_llm_provider != "reducto": + raise ValueError("unqualified Reducto models require custom_llm_provider='reducto'") + return self + + +_REDUCTO_PROVIDER_REJECTED_DOCUMENT: Final = ReductoDocumentUrlDocument( + type="document_url", + document_url="reducto://invalid-document-for-parity", +) +REDUCTO_V3_PROVIDER_REJECTED_INPUTS: Final[tuple[ReductoParseV3SdkInput, ...]] = ( + ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=_REDUCTO_PROVIDER_REJECTED_DOCUMENT, + ), +) +REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS: Final[tuple[ReductoParseLegacySdkInput, ...]] = ( + ReductoParseLegacySdkInput( + model="reducto/parse-legacy", + document=_REDUCTO_PROVIDER_REJECTED_DOCUMENT, + ), +) + + +_REDUCTO_API_BASE: Final = "https://platform.reducto.ai" + + +def _formatting_strategy() -> SearchStrategy[ReductoFormatting]: + values: Final = st.one_of( + st.sampled_from(("dynamic", "html", "md", "json", "csv", "jsonbbox")).map( + lambda value: {"table_output_format": value} + ), + st.sampled_from((False, True)).map(lambda value: {"add_page_markers": value}), + st.sampled_from((False, True)).map(lambda value: {"merge_tables": value}), + st.sampled_from( + ( + (), + ("hyperlinks",), + ("change_tracking", "highlight", "comments"), + ("signatures", "ignore_watermarks"), + ) + ) + .map(list) + .map(lambda value: {"include": value}), + ) + return values.map(ReductoFormatting.model_validate) + + +def _chunking_strategy() -> SearchStrategy[ReductoChunking]: + return st.one_of( + st.sampled_from(("disabled", "section", "page", "block", "page_sections")).map( + lambda mode: ReductoChunking(chunk_mode=mode) + ), + st.just(ReductoChunking(chunk_mode="variable")), + st.sampled_from((250, 1000, 1500)).map(lambda size: ReductoChunking(chunk_mode="variable", chunk_size=size)), + st.sampled_from((32, 128)).map( + lambda overlap: ReductoChunking(chunk_mode="variable", chunk_size=1000, chunk_overlap=overlap) + ), + ) + + +def _retrieval_strategy() -> SearchStrategy[ReductoRetrieval]: + filter_blocks: Final = cast( + SearchStrategy[list[ReductoBlockType]], + st.sampled_from(_REDUCTO_FILTER_BLOCK_GROUPS).map(list), + ) + return st.one_of( + _chunking_strategy().map(lambda chunking: ReductoRetrieval(chunking=chunking)), + filter_blocks.map(lambda selected_blocks: ReductoRetrieval(filter_blocks=selected_blocks)), + st.sampled_from((False, True)).map( + lambda optimized: ReductoRetrieval( + chunking=ReductoChunking(chunk_mode="variable"), + embedding_optimized=optimized, + ) + ), + ) + + +def _settings_strategy() -> SearchStrategy[ReductoSettings]: + # force_url_result stays model-compatible but is not recorded until the + # response transform follows and downloads result.url. + return_images: Final[SearchStrategy[list[ReductoReturnImage]]] = st.sampled_from(_REDUCTO_RETURN_IMAGE_GROUPS).map( + list + ) + page_ranges: Final = st.one_of( + st.just(ReductoPageRange(start=1, end=1)), + st.just(ReductoPageRange(start=1, end=3)), + st.sampled_from( + ( + ( + ReductoPageRange(start=1, end=2), + ReductoPageRange(start=4, end=5), + ), + ) + ).map(list), + ) + return st.one_of( + st.just(ReductoSettings(model="r-1")), + st.sampled_from(("standard", "legacy")).map(lambda value: ReductoSettings(ocr_system=value)), + st.sampled_from(("hybrid", "ocr", "metadata")).map(lambda value: ReductoSettings(extraction_mode=value)), + st.just(ReductoSettings(return_ocr_data=True)), + return_images.map(lambda selected_images: ReductoSettings(return_images=selected_images)), + st.just(ReductoSettings(embed_pdf_metadata=True)), + st.sampled_from((50, 100, 250)).map( + lambda dpi: ReductoSettings(embed_pdf_metadata=True, embed_pdf_metadata_dpi=dpi) + ), + st.just(ReductoSettings(timeout=300.0)), + page_ranges.map(lambda page_range: ReductoSettings(page_range=page_range)), + ) + + +def _reducto_v3_baseline( + route: _ReductoV3Route, + document: ReductoDocument, + inline_image_data_uri: str, +) -> ReductoParseV3SdkInput: + if route == "image": + inline_image: Final = ReductoImageUrlDocument.model_validate( + image_data_document(inline_image_data_uri).model_dump(mode="json") + ) + return ReductoParseV3SdkInput(model="reducto/parse-v3", document=inline_image) + if route == "unqualified": + return ReductoParseV3SdkInput( + model="parse-v3", + custom_llm_provider="reducto", + document=document, + ) + return ReductoParseV3SdkInput(model="reducto/parse-v3", document=document) + + +def reducto_v3_input_strategy( + inline_image_data_uri: str, + document: ReductoDocument | None = None, +) -> SearchStrategy[ReductoParseV3SdkInput]: + selected_document: Final = document or ReductoDocumentUrlDocument( + type="document_url", document_url="reducto://fixture-document.pdf" + ) + baseline_routes: Final[tuple[_ReductoV3Route, ...]] = ("qualified", "image", "unqualified") + return st.one_of( + st.sampled_from(baseline_routes).map( + lambda route: _reducto_v3_baseline(route, selected_document, inline_image_data_uri) + ), + _formatting_strategy().map( + lambda formatting: ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=selected_document, + formatting=formatting, + ) + ), + _retrieval_strategy().map( + lambda retrieval: ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=selected_document, + retrieval=retrieval, + ) + ), + _settings_strategy().map( + lambda settings: ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=selected_document, + settings=settings, + ) + ), + ) + + +def _reducto_legacy_input( + route: _ReductoLegacyRoute, + document: ReductoDocument, +) -> ReductoParseLegacySdkInput: + if route == "unqualified": + return ReductoParseLegacySdkInput( + model="parse-legacy", + custom_llm_provider="reducto", + document=document, + ) + return ReductoParseLegacySdkInput(model="reducto/parse-legacy", document=document) + + +def reducto_legacy_input_strategy( + document: ReductoDocument | None = None, +) -> SearchStrategy[ReductoParseLegacySdkInput]: + selected_document: Final = document or ReductoDocumentUrlDocument( + type="document_url", document_url="reducto://fixture-document.pdf" + ) + routes: Final[tuple[_ReductoLegacyRoute, ...]] = ("qualified", "unqualified") + return st.sampled_from(routes).map(lambda route: _reducto_legacy_input(route, selected_document)) + + +def reducto_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str +) -> tuple[OcrRecordingTarget, ...]: + api_key: Final = environ.get("REDUCTO_API_KEY") + if not api_key: + return () + base_url: Final = environ.get("REDUCTO_API_BASE", _REDUCTO_API_BASE).rstrip("/") + document: Final = ReductoDocumentUrlDocument(type="document_url", document_url=structured_pdf_data_uri()) + invocation: Final = invoke_with_api_key(client, api_key) + return ( + OcrRecordingTarget( + name="reducto-v3", + upstream=UpstreamEndpoint(base_url=base_url), + strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_v3_input_strategy(inline_image_data_uri, document)), + invocation=invocation, + required_inputs=REDUCTO_V3_PROVIDER_REJECTED_INPUTS, + ), + OcrRecordingTarget( + name="reducto-legacy", + upstream=UpstreamEndpoint(base_url=base_url), + strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_legacy_input_strategy(document)), + invocation=invocation, + required_inputs=REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS, + ), + ) diff --git a/tests/test_litellm/ocr/fixtures/vertex.py b/tests/test_litellm/ocr/fixtures/vertex.py new file mode 100644 index 00000000000..f4e51fb1cd2 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/vertex.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final, Literal, cast + +from hypothesis import strategies as st +from hypothesis.strategies import DrawFn, SearchStrategy + +from tests.route_parity.fixtures.recording import UpstreamEndpoint +from tests.test_litellm.ocr.fixtures.base import OcrDocument, OcrSdkInputBase +from tests.test_litellm.ocr.fixtures.common import ( + OcrFixtureClient, + OcrRecordingTarget, + image_data_document, + invoke_with_api_key, +) +from tests.test_litellm.ocr.fixtures.mistral import ( + MistralCompatibleOcrSdkInput, + mistral_input_values_strategy, +) + +VertexMistralModel = Literal["vertex_ai/mistral-ocr-2505"] +VertexDeepSeekModel = Literal["vertex_ai/deepseek-ai/deepseek-ocr-maas"] +VertexMistralFixtureModel = VertexMistralModel | Literal["vertex_ai/invalid-ocr-model-for-parity"] +VertexDeepSeekFixtureModel = VertexDeepSeekModel | Literal["vertex_ai/deepseek-ai/invalid-ocr-model-for-parity"] + +VERTEX_MISTRAL_MODELS: Final[tuple[VertexMistralModel, ...]] = ("vertex_ai/mistral-ocr-2505",) +VERTEX_DEEPSEEK_MODELS: Final[tuple[VertexDeepSeekModel, ...]] = ("vertex_ai/deepseek-ai/deepseek-ocr-maas",) + + +class VertexMistralOcrSdkInput(MistralCompatibleOcrSdkInput): + contract: Literal["vertex_mistral"] = "vertex_mistral" + model: VertexMistralFixtureModel = "vertex_ai/mistral-ocr-2505" + custom_llm_provider: Literal["vertex_ai"] | None = None + vertex_project: str + vertex_location: str = "us-central1" + + +class VertexDeepSeekOcrSdkInput(OcrSdkInputBase): + contract: Literal["vertex_deepseek"] = "vertex_deepseek" + model: VertexDeepSeekFixtureModel = "vertex_ai/deepseek-ai/deepseek-ocr-maas" + document: OcrDocument + custom_llm_provider: Literal["vertex_ai"] | None = None + vertex_project: str + vertex_location: str = "us-central1" + + +def vertex_mistral_provider_rejected_inputs( + project: str, + location: str, + inline_image_data_uri: str, +) -> tuple[VertexMistralOcrSdkInput, ...]: + return ( + VertexMistralOcrSdkInput( + model="vertex_ai/invalid-ocr-model-for-parity", + document=image_data_document(inline_image_data_uri), + vertex_project=project, + vertex_location=location, + ), + ) + + +def vertex_deepseek_provider_rejected_inputs( + project: str, + location: str, + inline_image_data_uri: str, +) -> tuple[VertexDeepSeekOcrSdkInput, ...]: + return ( + VertexDeepSeekOcrSdkInput( + model="vertex_ai/deepseek-ai/invalid-ocr-model-for-parity", + document=image_data_document(inline_image_data_uri), + vertex_project=project, + vertex_location=location, + ), + ) + + +def _as_vertex_mistral( + values: dict[str, object], + project: str, + location: str, + model: VertexMistralModel, +) -> VertexMistralOcrSdkInput: + return VertexMistralOcrSdkInput.model_validate( + {**values, "model": model, "vertex_project": project, "vertex_location": location} + ) + + +def vertex_mistral_input_strategy( + project: str, + location: str, + inline_image_data_uri: str, +) -> SearchStrategy[VertexMistralOcrSdkInput]: + return st.builds( + _as_vertex_mistral, + project=st.just(project), + location=st.just(location), + model=st.sampled_from(VERTEX_MISTRAL_MODELS), + values=mistral_input_values_strategy("2505", inline_image_data_uri), + ) + + +@st.composite +def vertex_deepseek_input_strategy( + draw: DrawFn, project: str, location: str, inline_image_data_uri: str +) -> VertexDeepSeekOcrSdkInput: + return VertexDeepSeekOcrSdkInput.model_validate( + { + "model": draw(st.sampled_from(VERTEX_DEEPSEEK_MODELS)), + "document": image_data_document(inline_image_data_uri), + "vertex_project": project, + "vertex_location": location, + } + ) + + +def vertex_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str +) -> tuple[OcrRecordingTarget, ...]: + api_key: Final = environ.get("VERTEX_AI_API_KEY") + project: Final = environ.get("VERTEXAI_PROJECT") or environ.get("VERTEX_PROJECT") + location: Final = environ.get("VERTEXAI_LOCATION") or environ.get("VERTEX_LOCATION") or "us-central1" + if not api_key or not project: + return () + base_url: Final = environ.get("VERTEX_AI_API_BASE") or f"https://{location}-aiplatform.googleapis.com" + invocation: Final = invoke_with_api_key(client, api_key) + return ( + OcrRecordingTarget( + name="vertex-mistral", + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), + strategy=cast( + SearchStrategy[OcrSdkInputBase], + vertex_mistral_input_strategy(project, location, inline_image_data_uri), + ), + invocation=invocation, + required_inputs=vertex_mistral_provider_rejected_inputs(project, location, inline_image_data_uri), + ), + OcrRecordingTarget( + name="vertex-deepseek", + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), + strategy=cast( + SearchStrategy[OcrSdkInputBase], + vertex_deepseek_input_strategy(project, location, inline_image_data_uri), + ), + invocation=invocation, + required_inputs=vertex_deepseek_provider_rejected_inputs(project, location, inline_image_data_uri), + ), + ) diff --git a/tests/test_litellm/ocr/test_fixture_models.py b/tests/test_litellm/ocr/test_fixture_models.py new file mode 100644 index 00000000000..12e88de5d1e --- /dev/null +++ b/tests/test_litellm/ocr/test_fixture_models.py @@ -0,0 +1,1388 @@ +from __future__ import annotations + +import base64 +from collections.abc import Callable +from datetime import date +from pathlib import Path +from typing import Final, TypeVar, cast +from unittest.mock import patch +from urllib.parse import parse_qs, urlparse + +import httpx +import pytest +import respx +from hypothesis import find, given, settings +from hypothesis import strategies as st +from hypothesis.strategies import DataObject, SearchStrategy +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError + +from litellm.llms.azure_ai.ocr.document_intelligence.transformation import AzureDocumentIntelligenceOCRConfig +from litellm.llms.azure_ai.ocr.transformation import AzureAIOCRConfig +from litellm.llms.base_llm.ocr.transformation import ( + BaseOCRConfig, + DocumentType, + OCRRequestData, +) +from litellm.llms.mistral.ocr.transformation import MistralOCRConfig +from litellm.llms.reducto.ocr.transformation import ReductoParseLegacyConfig, ReductoParseV3Config +from litellm.llms.vertex_ai.ocr.deepseek_transformation import VertexAIDeepSeekOCRConfig +from litellm.llms.vertex_ai.ocr.transformation import VertexAIOCRConfig +from tests.route_parity.fixtures.media import structured_pdf_data_uri +from tests.test_litellm.ocr.conftest import ocr_fixture_marks +from tests.test_litellm.ocr.fixtures.azure import ( + AZURE_DOCUMENT_INTELLIGENCE_MODELS, + AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS, + AZURE_MISTRAL_MODELS, + AzureDocumentIntelligenceOcrSdkInput, + AzureMistralOcrSdkInput, + azure_document_intelligence_input_strategy, + azure_mistral_input_strategy, +) +from tests.test_litellm.ocr.fixtures.base import ( + DocumentUrlDocument, + ImageUrlDocument, + ImageUrlValue, + JsonSchemaDefinition, + JsonSchemaResponseFormat, + OcrSdkInputBase, +) +from tests.test_litellm.ocr.fixtures.mistral import MISTRAL_MODELS, MistralOcrSdkInput, mistral_input_strategy +from tests.test_litellm.ocr.fixtures.models import OcrParityCase, OcrSdkInput +from tests.test_litellm.ocr.fixtures.reducto import ( + REDUCTO_LEGACY_MODELS, + REDUCTO_V3_MODELS, + ReductoChunking, + ReductoDocumentUrlDocument, + ReductoFormatting, + ReductoImageUrlDocument, + ReductoPageRange, + ReductoParseLegacySdkInput, + ReductoParseV3SdkInput, + ReductoRetrieval, + ReductoSettings, + reducto_legacy_input_strategy, + reducto_v3_input_strategy, +) +from tests.test_litellm.ocr.fixtures.vertex import ( + VERTEX_DEEPSEEK_MODELS, + VERTEX_MISTRAL_MODELS, + VertexDeepSeekOcrSdkInput, + VertexMistralOcrSdkInput, + vertex_deepseek_input_strategy, + vertex_mistral_input_strategy, +) + +COMMON_FIELDS: Final = frozenset( + {"contract", "model", "document", "custom_llm_provider", "vertex_project", "vertex_location"} +) +SUPPORTED_OCR_PROVIDERS: Final = frozenset({"mistral", "azure_ai", "reducto", "vertex_ai"}) +ACTIVE_OCR_MODELS: Final = frozenset( + ( + *MISTRAL_MODELS, + *AZURE_MISTRAL_MODELS, + *AZURE_DOCUMENT_INTELLIGENCE_MODELS, + *VERTEX_MISTRAL_MODELS, + *VERTEX_DEEPSEEK_MODELS, + *REDUCTO_V3_MODELS, + *REDUCTO_LEGACY_MODELS, + ) +) +_MISTRAL_2512_OR_NEWER: Final = frozenset( + { + "mistral/mistral-ocr-2512", + "mistral/mistral-ocr-3", + "mistral/mistral-ocr-3-0", + "mistral/mistral-ocr-4", + "mistral/mistral-ocr-4-0", + "mistral/mistral-ocr-4-1", + "mistral/mistral-ocr-latest", + } +) +_MISTRAL_4_OR_NEWER: Final = frozenset( + { + "mistral/mistral-ocr-4", + "mistral/mistral-ocr-4-0", + "mistral/mistral-ocr-4-1", + "mistral/mistral-ocr-latest", + } +) +_MISTRAL_OPTION_GROUPS: Final = frozenset( + { + frozenset[str](), + *( + frozenset({field}) + for field in ( + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "extract_header", + "extract_footer", + "table_format", + "confidence_scores_granularity", + "include_blocks", + ) + ), + frozenset({"document_annotation_format", "document_annotation_prompt"}), + frozenset({"include_blocks", "confidence_scores_granularity"}), + } +) +_MISTRAL_2505_OPTION_GROUPS: Final = frozenset( + { + frozenset[str](), + *( + frozenset({field}) + for field in ( + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "confidence_scores_granularity", + ) + ), + frozenset({"document_annotation_format", "document_annotation_prompt"}), + } +) +_AZURE_MISTRAL_OPTION_GROUPS: Final = _MISTRAL_2505_OPTION_GROUPS - { + frozenset({"document_annotation_format", "document_annotation_prompt"}) +} +_REDUCTO_FORMATTING_INCLUDE_GROUPS: Final = ( + (), + ("hyperlinks",), + ("change_tracking", "highlight", "comments"), + ("signatures", "ignore_watermarks"), +) +_REDUCTO_FILTER_BLOCK_GROUPS: Final = ( + (), + ("Header",), + ("Header", "Footer", "Page Number"), + ("Figure", "Table", "Key Value"), +) +_REDUCTO_RETURN_IMAGE_GROUPS: Final = ( + (), + ("figure",), + ("table",), + ("page",), + ("figure", "table"), +) +_FIND_SETTINGS: Final = settings(max_examples=2_000, deadline=None, derandomize=True, database=None) +_FixtureInputT = TypeVar("_FixtureInputT") +INLINE_IMAGE_DATA_URI: Final = "data:image/png;base64,dGVzdA==" +_MapOcrParams = Callable[[dict[str, object], dict[str, object], str], dict[str, object]] +_TransformOcrRequest = Callable[ + [str, DocumentType, dict[str, object], dict[str, object]], + OCRRequestData, +] +_GetCompleteUrl = Callable[[str | None, str, dict[str, object]], str] + + +def _transform_with_stubbed_download( + transform_request: _TransformOcrRequest, + model: str, + document: DocumentType, + mapped: dict[str, object], +) -> OCRRequestData: + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + source: Final = document[source_key] + if source.startswith("data:"): + return transform_request(model, document, mapped, {}) + media_type: Final = "image/png" if document["type"] == "image_url" else "application/pdf" + with respx.mock(assert_all_called=False) as router: + router.route(method="GET").mock( + return_value=httpx.Response(200, content=b"\x00", headers={"content-type": media_type}) + ) + return transform_request(model, document, mapped, {}) + + +def _find_fixture( + strategy: SearchStrategy[_FixtureInputT], + predicate: Callable[[_FixtureInputT], bool], +) -> _FixtureInputT: + return find(strategy, predicate, settings=_FIND_SETTINGS) + + +def _document_transport(document: ImageUrlDocument | DocumentUrlDocument) -> tuple[str, str]: + if isinstance(document, ImageUrlDocument): + source: Final = document.image_url.url if isinstance(document.image_url, ImageUrlValue) else document.image_url + return document.type, "data" if source.startswith("data:") else "remote" + return document.type, "data" if document.document_url.startswith("data:") else "remote" + + +def _normalized_azure_pages(pages: object) -> str: + if isinstance(pages, str): + return pages.replace(" ", "") + assert isinstance(pages, list) + raw_pages: Final = cast(list[object], pages) + if all(isinstance(page, int) for page in raw_pages): + integer_pages: Final = cast(list[int], raw_pages) + return ",".join(str(page + 1) for page in sorted(set(integer_pages))) + string_pages: Final = cast(list[str], raw_pages) + return ",".join(page.strip() for page in string_pages) + + +def test_structured_pdf_exercises_semantic_ocr_features() -> None: + encoded: Final = structured_pdf_data_uri().partition(",")[2] + pdf: Final = base64.b64decode(encoded, validate=True) + + assert pdf.startswith(b"%PDF-1.") + assert b"/Count 5" in pdf + assert pdf.count(b"/Subtype /Image") == 3 + assert all( + marker in pdf + for marker in ( + b"/Width 120", + b"/Width 320", + b"/Width 360", + b"/Subtype /Highlight", + b"/Subtype /Link", + b"/Subtype /Text", + b"/Title (Quarterly Operations Report)", + ) + ) + assert b"Invoice Number: INV-2048" in pdf + assert b"Formula: gross margin" in pdf + assert b"Approved by: Jordan Lee" in pdf + + +class _ModelRegistryEntry(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + mode: str | None = None + litellm_provider: str | None = None + deprecation_date: date | None = None + + +MODEL_REGISTRY: Final = TypeAdapter(dict[str, dict[str, JsonValue]]) + + +def _provider_fields(model: type[OcrSdkInputBase]) -> set[str]: + return set(model.model_fields) - COMMON_FIELDS + + +def _supported_params(config: BaseOCRConfig, model: str) -> set[str]: + get_supported_params: Final = cast(Callable[[str], list[str]], config.get_supported_ocr_params) + return set(get_supported_params(model)) + + +def _mistral_input(**params: object) -> MistralOcrSdkInput: + return MistralOcrSdkInput.model_validate( + { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "image_url", "image_url": "https://example.com/image.png"}, + **params, + } + ) + + +def _reducto_document() -> ReductoDocumentUrlDocument: + return ReductoDocumentUrlDocument( + type="document_url", + document_url="reducto://fixture-document.pdf", + ) + + +def test_fixture_catalogs_match_active_registered_ocr_models() -> None: + registry_path: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" + registry: Final = MODEL_REGISTRY.validate_json(registry_path.read_text(encoding="utf-8")) + active_registered: Final = frozenset( + model + for model, raw_metadata in registry.items() + if raw_metadata.get("mode") == "ocr" and raw_metadata.get("litellm_provider") in SUPPORTED_OCR_PROVIDERS + for metadata in (_ModelRegistryEntry.model_validate(raw_metadata),) + if metadata.deprecation_date is None or metadata.deprecation_date > date.today() + ) + + assert ACTIVE_OCR_MODELS == active_registered + + +@pytest.mark.parametrize( + ("fixture_model", "provider_config", "model"), + ( + (MistralOcrSdkInput, MistralOCRConfig(), "mistral-ocr-latest"), + (AzureMistralOcrSdkInput, AzureAIOCRConfig(), "mistral-document-ai-2512"), + ( + AzureDocumentIntelligenceOcrSdkInput, + AzureDocumentIntelligenceOCRConfig(), + "doc-intelligence/prebuilt-layout", + ), + (VertexMistralOcrSdkInput, VertexAIOCRConfig(), "mistral-ocr-2505"), + (VertexDeepSeekOcrSdkInput, VertexAIDeepSeekOCRConfig(), "deepseek-ai/deepseek-ocr-maas"), + (ReductoParseV3SdkInput, ReductoParseV3Config(), "parse-v3"), + (ReductoParseLegacySdkInput, ReductoParseLegacyConfig(), "parse-legacy"), + ), +) +def test_fixture_fields_match_provider_config( + fixture_model: type[OcrSdkInputBase], provider_config: BaseOCRConfig, model: str +) -> None: + assert _provider_fields(fixture_model) == _supported_params(provider_config, model) + + +@pytest.mark.parametrize( + "sdk_input", + ( + AzureMistralOcrSdkInput( + model="azure_ai/mistral-document-ai-2512", + document=ImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + ), + VertexMistralOcrSdkInput( + document=ImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + vertex_project="project-1", + ), + AzureDocumentIntelligenceOcrSdkInput( + model="azure_ai/doc-intelligence/prebuilt-layout", + document=ImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + ), + VertexDeepSeekOcrSdkInput( + document=ImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + vertex_project="project-1", + ), + ), +) +def test_provider_contract_is_explicit_but_not_forwarded(sdk_input: OcrSdkInput) -> None: + assert sdk_input.canonical_input()["contract"] == sdk_input.contract + assert "contract" not in sdk_input.as_sdk_kwargs() + + +@pytest.mark.parametrize("legacy_key", ("boundary", None)) +def test_ocr_parity_case_migrates_legacy_contract_metadata(legacy_key: str | None) -> None: + litellm_input: Final[dict[str, object]] = { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "document_url", "document_url": "https://example.com/document.pdf"}, + } + if legacy_key is not None: + litellm_input[legacy_key] = "mistral" + + fixture: Final = OcrParityCase.model_validate({"litellm_input": litellm_input, "provider_responses": ()}) + + assert fixture.litellm_input.contract == "mistral" + + +def test_mistral_input_preserves_omission_and_explicit_boolean_values() -> None: + omitted: Final = _mistral_input().as_sdk_kwargs() + explicit: Final = _mistral_input(extract_header=False, include_blocks=True).as_sdk_kwargs() + + assert "extract_header" not in omitted + assert "include_blocks" not in omitted + assert explicit["extract_header"] is False + assert explicit["include_blocks"] is True + + +def test_mistral_input_supports_document_and_page_variants() -> None: + nested_image: Final = MistralOcrSdkInput( + model="mistral/mistral-ocr-4-1", + document=ImageUrlDocument( + type="image_url", + image_url=ImageUrlValue(url="https://example.com/image.png", detail="high"), + ), + pages="0,2-4", + ) + named_document: Final = MistralOcrSdkInput( + model="mistral/mistral-ocr-2512", + document=DocumentUrlDocument( + type="document_url", + document_url="https://example.com/document.pdf", + document_name="invoice.pdf", + ), + ) + + assert nested_image.canonical_input()["document"] == { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png", "detail": "high"}, + } + assert nested_image.as_sdk_kwargs()["pages"] == "0,2-4" + assert named_document.canonical_input()["document"] == { + "type": "document_url", + "document_url": "https://example.com/document.pdf", + "document_name": "invoice.pdf", + } + + +def test_mistral_annotation_schema_serializes_provider_alias() -> None: + annotation: Final = JsonSchemaResponseFormat( + type="json_schema", + json_schema=JsonSchemaDefinition( + name="invoice", + schema={"type": "object"}, + ), + ) + sdk_input: Final = _mistral_input( + document_annotation_format=annotation, + document_annotation_prompt="Extract invoice fields", + ) + + assert sdk_input.canonical_input()["document_annotation_format"] == { + "type": "json_schema", + "json_schema": { + "name": "invoice", + "schema": {"type": "object"}, + }, + } + + +def test_mistral_annotation_prompt_requires_format() -> None: + with pytest.raises(ValidationError, match="requires document_annotation_format"): + _mistral_input(document_annotation_prompt="Extract invoice fields") + + +@pytest.mark.parametrize("field", ("extract_header", "extract_footer", "include_blocks")) +def test_mistral_nonnullable_booleans_reject_null(field: str) -> None: + with pytest.raises(ValidationError): + _mistral_input(**{field: None}) + + +def test_unqualified_models_require_explicit_provider() -> None: + with pytest.raises(ValidationError, match="custom_llm_provider='mistral'"): + MistralOcrSdkInput( + model="mistral-ocr-latest", + document=ImageUrlDocument(type="image_url", image_url="https://example.com/image.png"), + ) + with pytest.raises(ValidationError, match="custom_llm_provider='reducto'"): + ReductoParseV3SdkInput(model="parse-v3", document=_reducto_document()) + + +@pytest.mark.parametrize("model", tuple(model.removeprefix("mistral/") for model in MISTRAL_MODELS)) +def test_unqualified_mistral_models_accept_explicit_provider(model: str) -> None: + sdk_input: Final = MistralOcrSdkInput.model_validate( + { + "model": model, + "custom_llm_provider": "mistral", + "document": ImageUrlDocument(type="image_url", image_url="https://example.com/image.png"), + } + ) + + assert sdk_input.model == model + + +@pytest.mark.parametrize( + ("model", "model_type"), + (("parse-v3", ReductoParseV3SdkInput), ("parse-legacy", ReductoParseLegacySdkInput)), +) +def test_unqualified_reducto_models_accept_explicit_provider( + model: str, model_type: type[ReductoParseV3SdkInput] | type[ReductoParseLegacySdkInput] +) -> None: + sdk_input: Final = model_type.model_validate( + {"model": model, "custom_llm_provider": "reducto", "document": _reducto_document()} + ) + + assert sdk_input.model == model + + +@pytest.mark.parametrize( + "document", + ( + {"type": "image_url", "image_url": "data:image/png;base64,AA=="}, + {"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + ), +) +def test_vertex_deepseek_request_maps_both_document_types_to_image_content( + document: DocumentType, +) -> None: + request: Final = VertexAIDeepSeekOCRConfig().transform_ocr_request( # pyright: ignore[reportUnknownMemberType] + model="deepseek-ai/deepseek-ocr-maas", + document=document, + optional_params={}, + headers={}, + ) + + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + data: Final = cast(dict[str, object], request.data) + messages: Final = cast(list[dict[str, object]], data["messages"]) + content: Final = cast(list[dict[str, object]], messages[0]["content"]) + assert content == [{"type": "image_url", "image_url": document[source_key]}] + + +@pytest.mark.parametrize( + "sdk_input", + ( + ReductoParseV3SdkInput(model="reducto/parse-v3", document=_reducto_document()), + ReductoParseLegacySdkInput(model="reducto/parse-legacy", document=_reducto_document()), + ), +) +def test_reducto_parity_cases_are_non_strict_xfails( + sdk_input: ReductoParseV3SdkInput | ReductoParseLegacySdkInput, +) -> None: + marks: Final = ocr_fixture_marks(OcrParityCase(litellm_input=sdk_input, provider_responses=())) + + assert len(marks) == 1 + assert marks[0].mark.name == "xfail" + assert marks[0].mark.kwargs["strict"] is False + + +def test_supported_parity_cases_have_no_marks() -> None: + sdk_input: Final = _mistral_input() + + assert ocr_fixture_marks(OcrParityCase(litellm_input=sdk_input, provider_responses=())) == () + + +def test_reducto_v3_preserves_nested_provider_params() -> None: + sdk_input: Final = ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=_reducto_document(), + formatting=ReductoFormatting(table_output_format="html", include=["hyperlinks"]), + retrieval=ReductoRetrieval(chunking=ReductoChunking(chunk_mode="variable", chunk_size=250, chunk_overlap=32)), + settings=ReductoSettings(embed_pdf_metadata=True, embed_pdf_metadata_dpi=250, page_range=[1, 3]), + ) + + assert sdk_input.as_sdk_kwargs()["formatting"] == { + "table_output_format": "html", + "include": ["hyperlinks"], + } + assert sdk_input.as_sdk_kwargs()["retrieval"] == { + "chunking": {"chunk_mode": "variable", "chunk_size": 250, "chunk_overlap": 32} + } + assert sdk_input.as_sdk_kwargs()["settings"] == { + "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 250, + "page_range": [1, 3], + } + + +def test_reducto_optional_objects_reject_explicit_null() -> None: + with pytest.raises(ValidationError): + ReductoParseV3SdkInput.model_validate( + { + "model": "reducto/parse-v3", + "document": _reducto_document(), + "formatting": None, + } + ) + + +@pytest.mark.parametrize( + "source", + ( + "https://example.com/document.pdf", + "not-a-document", + "data:application/pdf,not-base64", + "data:application/pdf;base64,not!base64", + ), +) +def test_reducto_document_rejects_unsupported_sources(source: str) -> None: + with pytest.raises(ValidationError): + ReductoDocumentUrlDocument(type="document_url", document_url=source) + + +def test_reducto_nested_constraints() -> None: + with pytest.raises(ValidationError, match="less than chunk_size"): + ReductoChunking(chunk_mode="variable", chunk_size=100, chunk_overlap=100) + with pytest.raises(ValidationError, match="greater than or equal to start"): + ReductoPageRange(start=3, end=2) + with pytest.raises(ValidationError): + ReductoSettings(embed_pdf_metadata_dpi=49) + with pytest.raises(ValidationError, match="must be unique"): + ReductoFormatting(include=["hyperlinks", "hyperlinks"]) + + +@settings(max_examples=100, deadline=None) +@given(model=st.sampled_from(MISTRAL_MODELS), data=st.data()) +def test_mistral_strategy_only_generates_bounded_valid_sdk_inputs(model: str, data: DataObject) -> None: + sdk_input: Final = data.draw(mistral_input_strategy(model, INLINE_IMAGE_DATA_URI)) + assert MistralOcrSdkInput.model_validate(sdk_input.canonical_input()) == sdk_input + optional_fields: Final = frozenset(sdk_input.model_fields_set) - {"model", "document"} + assert optional_fields in _MISTRAL_OPTION_GROUPS + if sdk_input.pages is not None: + assert sdk_input.pages in ([0], [0, 1], "0-2") + if sdk_input.image_limit is not None: + assert sdk_input.image_limit == 1 + if sdk_input.image_min_size is not None: + assert sdk_input.image_min_size == 300 + if sdk_input.table_format is not None: + assert sdk_input.table_format in {"markdown", "html"} + if sdk_input.confidence_scores_granularity is not None: + assert sdk_input.confidence_scores_granularity in {"page", "word", "block"} + if sdk_input.confidence_scores_granularity == "block": + assert sdk_input.include_blocks is True + if model not in _MISTRAL_2512_OR_NEWER: + assert optional_fields.isdisjoint({"extract_header", "extract_footer", "table_format"}) + if model not in _MISTRAL_4_OR_NEWER: + assert "include_blocks" not in optional_fields + assert not isinstance(sdk_input.pages, str) + if optional_fields: + assert isinstance(sdk_input.document, DocumentUrlDocument) + assert sdk_input.document.document_url == structured_pdf_data_uri() + + +@pytest.mark.parametrize( + "transport", + ( + ("image_url", "remote"), + ("image_url", "data"), + ("document_url", "remote"), + ("document_url", "data"), + ), +) +def test_mistral_strategy_reaches_every_document_transform_branch(transport: tuple[str, str]) -> None: + sdk_input: Final = _find_fixture( + mistral_input_strategy("mistral/mistral-ocr-4-1", INLINE_IMAGE_DATA_URI), + lambda candidate: _document_transport(candidate.document) == transport, + ) + + assert _document_transport(sdk_input.document) == transport + + +@settings(max_examples=100, deadline=None) +@given(sdk_input=mistral_input_strategy("mistral/mistral-ocr-4-1", INLINE_IMAGE_DATA_URI)) +def test_mistral_strategy_values_survive_the_request_transform(sdk_input: MistralOcrSdkInput) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = {name: value for name, value in sdk_kwargs.items() if name not in {"model", "document"}} + config: Final = MistralOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + request: Final = transform_request(model, document, mapped, {}) + request_data: Final = cast(dict[str, object], request.data) + + assert mapped == optional_params + assert request_data == {"model": model, "document": document, **optional_params} + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("pages", [0]), + ("pages", [0, 1]), + ("pages", "0-2"), + ("include_image_base64", False), + ("include_image_base64", True), + ("image_limit", 1), + ("image_min_size", 300), + ("extract_header", False), + ("extract_header", True), + ("extract_footer", False), + ("extract_footer", True), + ("table_format", "markdown"), + ("table_format", "html"), + ("confidence_scores_granularity", "page"), + ("confidence_scores_granularity", "word"), + ("confidence_scores_granularity", "block"), + ("include_blocks", False), + ("include_blocks", True), + ), +) +def test_mistral_strategy_reaches_every_finite_scalar_value(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + mistral_input_strategy("mistral/mistral-ocr-4-1", INLINE_IMAGE_DATA_URI), + lambda candidate: field in candidate.model_fields_set and getattr(candidate, field) == value, + ) + + assert getattr(sdk_input, field) == value + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI)) +def test_reducto_v3_strategy_only_generates_bounded_valid_sdk_inputs(sdk_input: ReductoParseV3SdkInput) -> None: + assert ReductoParseV3SdkInput.model_validate(sdk_input.canonical_input()) == sdk_input + option_groups: Final = frozenset(sdk_input.model_fields_set) & {"formatting", "retrieval", "settings"} + assert len(option_groups) <= 1 + if "formatting" in option_groups: + formatting_fields: Final = frozenset(sdk_input.formatting.model_fields_set) + assert len(formatting_fields) == 1 + if "table_output_format" in formatting_fields: + assert sdk_input.formatting.table_output_format in {"dynamic", "html", "md", "json", "csv", "jsonbbox"} + if "add_page_markers" in formatting_fields: + assert sdk_input.formatting.add_page_markers in {False, True} + if "merge_tables" in formatting_fields: + assert sdk_input.formatting.merge_tables in {False, True} + if "include" in formatting_fields: + assert tuple(sdk_input.formatting.include) in _REDUCTO_FORMATTING_INCLUDE_GROUPS + if "retrieval" in option_groups: + retrieval_fields: Final = frozenset(sdk_input.retrieval.model_fields_set) + assert retrieval_fields in { + frozenset({"chunking"}), + frozenset({"filter_blocks"}), + frozenset({"chunking", "embedding_optimized"}), + } + chunking: Final = sdk_input.retrieval.chunking + if "chunking" in retrieval_fields: + assert chunking.chunk_mode in {"variable", "section", "page", "disabled", "block", "page_sections"} + assert chunking.chunk_size in {None, 250, 1000, 1500} + assert chunking.chunk_overlap in {0, 32, 128} + if chunking.chunk_size is not None or chunking.chunk_overlap: + assert chunking.chunk_mode == "variable" + if chunking.chunk_overlap: + assert chunking.chunk_size == 1000 + if "filter_blocks" in retrieval_fields: + assert tuple(sdk_input.retrieval.filter_blocks) in _REDUCTO_FILTER_BLOCK_GROUPS + if "embedding_optimized" in retrieval_fields: + assert chunking.chunk_mode == "variable" + assert chunking.chunk_size is None + assert chunking.chunk_overlap == 0 + assert sdk_input.retrieval.embedding_optimized in {False, True} + if "settings" in option_groups: + settings_fields: Final = frozenset(sdk_input.settings.model_fields_set) + assert settings_fields in { + frozenset({"model"}), + frozenset({"ocr_system"}), + frozenset({"extraction_mode"}), + frozenset({"return_ocr_data"}), + frozenset({"return_images"}), + frozenset({"embed_pdf_metadata"}), + frozenset({"embed_pdf_metadata", "embed_pdf_metadata_dpi"}), + frozenset({"timeout"}), + frozenset({"page_range"}), + } + assert settings_fields.isdisjoint( + { + "force_url_result", + "force_file_extension", + "persist_results", + "tenant_throttling", + "document_password", + "hybrid_vpc", + } + ) + if "model" in settings_fields: + assert sdk_input.settings.model == "r-1" + if "ocr_system" in settings_fields: + assert sdk_input.settings.ocr_system in {"standard", "legacy"} + if "extraction_mode" in settings_fields: + assert sdk_input.settings.extraction_mode in {"hybrid", "ocr", "metadata"} + if "return_ocr_data" in settings_fields: + assert sdk_input.settings.return_ocr_data is True + if "return_images" in settings_fields: + assert tuple(sdk_input.settings.return_images) in _REDUCTO_RETURN_IMAGE_GROUPS + if "embed_pdf_metadata_dpi" in settings_fields: + assert sdk_input.settings.embed_pdf_metadata is True + assert sdk_input.settings.embed_pdf_metadata_dpi in {50, 100, 250} + if "timeout" in settings_fields: + assert sdk_input.settings.timeout == 300.0 + if sdk_input.settings.page_range is not None: + dumped_range: Final = cast( + dict[str, object], sdk_input.settings.model_dump(mode="json", exclude_unset=True) + )["page_range"] + assert dumped_range in ( + {"start": 1, "end": 1}, + {"start": 1, "end": 3}, + [{"start": 1, "end": 2}, {"start": 4, "end": 5}], + ) + + +@settings(max_examples=60, deadline=None) +@given(sdk_input=reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI)) +def test_reducto_v3_strategy_values_survive_the_request_transform(sdk_input: ReductoParseV3SdkInput) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = { + name: value for name, value in sdk_kwargs.items() if name not in {"model", "document", "custom_llm_provider"} + } + config: Final = ReductoParseV3Config() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + + with patch.object(config, "_ensure_file_id_sync", return_value="reducto://fixture-document.pdf"): + request: Final = transform_request(model, document, mapped, {}) + + assert mapped == optional_params + assert cast(dict[str, object], request.data) == { + "input": "reducto://fixture-document.pdf", + **optional_params, + } + + +def test_reducto_v3_strategy_reaches_image_upload_branch_without_options() -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: isinstance(candidate.document, ReductoImageUrlDocument), + ) + + assert isinstance(sdk_input.document, ReductoImageUrlDocument) + assert sdk_input.document.image_url.startswith("data:image/") + assert sdk_input.model_fields_set == {"model", "document"} + + +@pytest.mark.parametrize( + ("model", "provider"), + (("reducto/parse-v3", None), ("parse-v3", "reducto")), +) +def test_reducto_v3_strategy_reaches_every_routing_form(model: str, provider: str | None) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: candidate.model == model and candidate.custom_llm_provider == provider, + ) + + assert sdk_input.model == model + assert sdk_input.custom_llm_provider == provider + + +@pytest.mark.parametrize("table_format", ("dynamic", "html", "md", "json", "csv", "jsonbbox")) +def test_reducto_v3_strategy_reaches_every_table_format(table_format: str) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "formatting" in candidate.model_fields_set + and "table_output_format" in candidate.formatting.model_fields_set + and candidate.formatting.table_output_format == table_format + ), + ) + + assert sdk_input.formatting.table_output_format == table_format + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("add_page_markers", False), + ("add_page_markers", True), + ("merge_tables", False), + ("merge_tables", True), + ), +) +def test_reducto_v3_strategy_reaches_every_formatting_boolean(field: str, value: bool) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "formatting" in candidate.model_fields_set + and field in candidate.formatting.model_fields_set + and getattr(candidate.formatting, field) is value + ), + ) + + assert getattr(sdk_input.formatting, field) is value + + +@pytest.mark.parametrize("include", _REDUCTO_FORMATTING_INCLUDE_GROUPS) +def test_reducto_v3_strategy_reaches_every_formatting_include(include: tuple[str, ...]) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "formatting" in candidate.model_fields_set + and "include" in candidate.formatting.model_fields_set + and tuple(candidate.formatting.include) == include + ), + ) + + assert tuple(sdk_input.formatting.include) == include + + +@pytest.mark.parametrize("chunk_mode", ("variable", "section", "page", "disabled", "block", "page_sections")) +def test_reducto_v3_strategy_reaches_every_chunk_mode(chunk_mode: str) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "retrieval" in candidate.model_fields_set + and "chunking" in candidate.retrieval.model_fields_set + and candidate.retrieval.chunking.chunk_mode == chunk_mode + ), + ) + + assert sdk_input.retrieval.chunking.chunk_mode == chunk_mode + + +@pytest.mark.parametrize("chunk_size", (250, 1000, 1500)) +def test_reducto_v3_strategy_reaches_every_chunk_size(chunk_size: int) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: candidate.retrieval.chunking.chunk_size == chunk_size, + ) + + assert sdk_input.retrieval.chunking.chunk_mode == "variable" + assert sdk_input.retrieval.chunking.chunk_size == chunk_size + + +@pytest.mark.parametrize("chunk_overlap", (32, 128)) +def test_reducto_v3_strategy_reaches_every_chunk_overlap(chunk_overlap: int) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: candidate.retrieval.chunking.chunk_overlap == chunk_overlap, + ) + + assert sdk_input.retrieval.chunking.chunk_mode == "variable" + assert sdk_input.retrieval.chunking.chunk_size == 1000 + assert sdk_input.retrieval.chunking.chunk_overlap == chunk_overlap + + +@pytest.mark.parametrize("filter_blocks", _REDUCTO_FILTER_BLOCK_GROUPS) +def test_reducto_v3_strategy_reaches_every_filter_block_group(filter_blocks: tuple[str, ...]) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "retrieval" in candidate.model_fields_set + and "filter_blocks" in candidate.retrieval.model_fields_set + and tuple(candidate.retrieval.filter_blocks) == filter_blocks + ), + ) + + assert tuple(sdk_input.retrieval.filter_blocks) == filter_blocks + + +@pytest.mark.parametrize("embedding_optimized", (False, True)) +def test_reducto_v3_strategy_reaches_every_embedding_setting(embedding_optimized: bool) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "retrieval" in candidate.model_fields_set + and "embedding_optimized" in candidate.retrieval.model_fields_set + and candidate.retrieval.embedding_optimized is embedding_optimized + ), + ) + + assert sdk_input.retrieval.chunking.chunk_mode == "variable" + assert sdk_input.retrieval.embedding_optimized is embedding_optimized + + +@pytest.mark.parametrize("dpi", (50, 100, 250)) +def test_reducto_v3_strategy_reaches_every_metadata_dpi(dpi: int) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "settings" in candidate.model_fields_set + and "embed_pdf_metadata_dpi" in candidate.settings.model_fields_set + and candidate.settings.embed_pdf_metadata_dpi == dpi + ), + ) + + assert sdk_input.settings.embed_pdf_metadata is True + assert sdk_input.settings.embed_pdf_metadata_dpi == dpi + + +def test_reducto_v3_strategy_reaches_metadata_with_default_dpi_omitted() -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "settings" in candidate.model_fields_set and candidate.settings.model_fields_set == {"embed_pdf_metadata"} + ), + ) + + assert sdk_input.settings.embed_pdf_metadata is True + assert "embed_pdf_metadata_dpi" not in sdk_input.settings.model_fields_set + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("model", "r-1"), + ("ocr_system", "standard"), + ("ocr_system", "legacy"), + ("extraction_mode", "hybrid"), + ("extraction_mode", "ocr"), + ("extraction_mode", "metadata"), + ("return_ocr_data", True), + ("timeout", 300.0), + ), +) +def test_reducto_v3_strategy_reaches_every_scalar_setting(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "settings" in candidate.model_fields_set + and field in candidate.settings.model_fields_set + and getattr(candidate.settings, field) == value + ), + ) + + assert getattr(sdk_input.settings, field) == value + + +@pytest.mark.parametrize("return_images", _REDUCTO_RETURN_IMAGE_GROUPS) +def test_reducto_v3_strategy_reaches_every_return_image_group(return_images: tuple[str, ...]) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "settings" in candidate.model_fields_set + and "return_images" in candidate.settings.model_fields_set + and tuple(candidate.settings.return_images) == return_images + ), + ) + + assert tuple(sdk_input.settings.return_images) == return_images + + +@pytest.mark.parametrize( + "page_range", + ( + {"start": 1, "end": 1}, + {"start": 1, "end": 3}, + [{"start": 1, "end": 2}, {"start": 4, "end": 5}], + ), +) +def test_reducto_v3_strategy_reaches_every_page_range_shape(page_range: object) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + cast( + dict[str, object], + candidate.settings.model_dump(mode="json", exclude_unset=True), + ).get("page_range") + == page_range + ), + ) + + assert sdk_input.settings.model_dump(mode="json", exclude_unset=True)["page_range"] == page_range + + +@settings(max_examples=10, deadline=None) +@given(sdk_input=reducto_legacy_input_strategy()) +def test_reducto_legacy_strategy_generates_valid_litellm_inputs(sdk_input: ReductoParseLegacySdkInput) -> None: + assert ReductoParseLegacySdkInput.model_validate(sdk_input.canonical_input()) == sdk_input + assert "enhance" not in sdk_input.model_fields_set + + +@settings(max_examples=10, deadline=None) +@given(sdk_input=reducto_legacy_input_strategy()) +def test_reducto_legacy_strategy_values_survive_the_request_transform( + sdk_input: ReductoParseLegacySdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + config: Final = ReductoParseLegacyConfig() + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + + with patch.object(config, "_ensure_file_id_sync", return_value="reducto://fixture-document.pdf"): + request: Final = transform_request(model, document, {}, {}) + + assert cast(dict[str, object], request.data) == { + "document_url": "reducto://fixture-document.pdf", + } + + +@pytest.mark.parametrize( + ("model", "provider"), + (("reducto/parse-legacy", None), ("parse-legacy", "reducto")), +) +def test_reducto_legacy_strategy_reaches_every_routing_form(model: str, provider: str | None) -> None: + sdk_input: Final = _find_fixture( + reducto_legacy_input_strategy(), + lambda candidate: candidate.model == model and candidate.custom_llm_provider == provider, + ) + + assert sdk_input.model == model + assert sdk_input.custom_llm_provider == provider + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI)) +def test_azure_mistral_strategy_is_contained_to_gateway_capabilities( + sdk_input: AzureMistralOcrSdkInput, +) -> None: + optional_fields: Final = frozenset(sdk_input.model_fields_set) - {"model", "document"} + + assert optional_fields in _AZURE_MISTRAL_OPTION_GROUPS + assert optional_fields.isdisjoint( + { + "document_annotation_prompt", + "extract_header", + "extract_footer", + "table_format", + "include_blocks", + "id", + } + ) + assert not isinstance(sdk_input.pages, str) + assert sdk_input.confidence_scores_granularity in {None, "page", "word"} + if optional_fields: + assert isinstance(sdk_input.document, DocumentUrlDocument) + assert sdk_input.document.document_url == structured_pdf_data_uri() + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("pages", [0]), + ("pages", [0, 1]), + ("include_image_base64", False), + ("include_image_base64", True), + ("image_limit", 1), + ("image_min_size", 300), + ("confidence_scores_granularity", "page"), + ("confidence_scores_granularity", "word"), + ), +) +def test_azure_mistral_strategy_reaches_every_gateway_scalar(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: field in candidate.model_fields_set and getattr(candidate, field) == value, + ) + + assert getattr(sdk_input, field) == value + + +@pytest.mark.parametrize("field", ("bbox_annotation_format", "document_annotation_format")) +def test_azure_mistral_strategy_reaches_every_gateway_schema(field: str) -> None: + sdk_input: Final = _find_fixture( + azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: frozenset(candidate.model_fields_set) - {"model", "document"} == frozenset({field}), + ) + + assert frozenset(sdk_input.model_fields_set) - {"model", "document"} == {field} + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI)) +def test_azure_mistral_strategy_exercises_url_conversion_and_inline_bypass( + sdk_input: AzureMistralOcrSdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = {name: value for name, value in sdk_kwargs.items() if name not in {"model", "document"}} + config: Final = AzureAIOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + + request: Final = _transform_with_stubbed_download(transform_request, model, document, mapped) + + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + source: Final = document[source_key] + expected_document: Final = dict(document) + if not source.startswith("data:"): + media_type: Final = "image/png" if document["type"] == "image_url" else "application/pdf" + expected_document[source_key] = f"data:{media_type};base64,AA==" + + assert mapped == optional_params + assert cast(dict[str, object], request.data) == { + "model": model, + "document": expected_document, + **optional_params, + } + + +@settings(max_examples=30, deadline=None) +@given(sdk_input=azure_document_intelligence_input_strategy()) +def test_azure_document_intelligence_strategy_only_generates_litellm_inputs( + sdk_input: AzureDocumentIntelligenceOcrSdkInput, +) -> None: + assert sdk_input.req_format == "litellm" + assert sdk_input.model in AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS + assert "contract" not in sdk_input.as_sdk_kwargs() + optional_fields: Final = frozenset(sdk_input.model_fields_set) - {"model", "document"} + assert optional_fields in { + frozenset[str](), + frozenset({"pages"}), + frozenset({"features"}), + frozenset({"pages", "features"}), + frozenset({"req_format"}), + } + if sdk_input.pages is not None: + assert sdk_input.pages in ([0], [2, 0, 0, 1], ["1", "2-4"], "1-4, 5", [0, 1]) + if isinstance(sdk_input.features, list): + assert tuple(sdk_input.features) in { + ("languages",), + ("ocrHighResolution",), + ("barcodes",), + ("formulas",), + ("styleFont",), + ("keyValuePairs",), + ("languages", "styleFont"), + } + if isinstance(sdk_input.features, str): + assert sdk_input.features == "languages, styleFont" + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=azure_document_intelligence_input_strategy()) +def test_azure_document_intelligence_strategy_exercises_request_transform( + sdk_input: AzureDocumentIntelligenceOcrSdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = {name: value for name, value in sdk_kwargs.items() if name not in {"model", "document"}} + config: Final = AzureDocumentIntelligenceOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + get_complete_url: Final = cast(_GetCompleteUrl, config.get_complete_url) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + url: Final = get_complete_url("https://document.example", model, mapped) + query: Final = parse_qs(urlparse(url).query) + request: Final = transform_request(model, document, mapped, {}) + + if sdk_input.pages is None: + assert "pages" not in mapped + assert "pages" not in query + else: + expected_pages: Final = _normalized_azure_pages(sdk_input.pages) + assert mapped["pages"] == expected_pages + assert query["pages"] == [expected_pages] + if sdk_input.features is None: + assert "features" not in mapped + assert "features" not in query + else: + raw_features: Final = ( + sdk_input.features.split(",") if isinstance(sdk_input.features, str) else sdk_input.features + ) + expected_features: Final = ",".join(feature.strip() for feature in raw_features) + assert mapped["features"] == expected_features + assert query["features"] == [expected_features] + + source: Final = document["document_url"] if document["type"] == "document_url" else document["image_url"] + assert isinstance(source, str) + expected_body: Final = ( + {"base64Source": source.partition(",")[2]} if source.startswith("data:") else {"urlSource": source} + ) + assert cast(dict[str, object], request.data) == expected_body + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("pages", [0]), + ("pages", [2, 0, 0, 1]), + ("pages", ["1", "2-4"]), + ("pages", "1-4, 5"), + ("features", ["languages"]), + ("features", ["ocrHighResolution"]), + ("features", ["barcodes"]), + ("features", ["formulas"]), + ("features", ["styleFont"]), + ("features", ["keyValuePairs"]), + ("features", "languages, styleFont"), + ), +) +def test_azure_document_intelligence_strategy_reaches_every_finite_value(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + azure_document_intelligence_input_strategy(), + lambda candidate: field in candidate.model_fields_set and getattr(candidate, field) == value, + ) + + assert getattr(sdk_input, field) == value + + +def test_azure_document_intelligence_strategy_reaches_combined_query_branch() -> None: + sdk_input: Final = _find_fixture( + azure_document_intelligence_input_strategy(), + lambda candidate: {"pages", "features"}.issubset(candidate.model_fields_set), + ) + + assert sdk_input.pages == [0, 1] + assert sdk_input.features == ["languages", "styleFont"] + + +@pytest.mark.parametrize( + "transport", + (("document_url", "data"), ("image_url", "remote")), +) +def test_azure_document_intelligence_strategy_reaches_body_source_branches( + transport: tuple[str, str], +) -> None: + sdk_input: Final = _find_fixture( + azure_document_intelligence_input_strategy(), + lambda candidate: _document_transport(candidate.document) == transport, + ) + + assert _document_transport(sdk_input.document) == transport + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI)) +def test_vertex_mistral_strategy_is_contained_to_2505_capabilities( + sdk_input: VertexMistralOcrSdkInput, +) -> None: + optional_fields: Final = frozenset(sdk_input.model_fields_set) - { + "model", + "document", + "vertex_project", + "vertex_location", + } + + assert optional_fields in _MISTRAL_2505_OPTION_GROUPS + assert optional_fields.isdisjoint({"extract_header", "extract_footer", "table_format", "include_blocks", "id"}) + assert not isinstance(sdk_input.pages, str) + assert sdk_input.confidence_scores_granularity in {None, "page", "word"} + if optional_fields: + assert isinstance(sdk_input.document, DocumentUrlDocument) + assert sdk_input.document.document_url == structured_pdf_data_uri() + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("pages", [0]), + ("pages", [0, 1]), + ("include_image_base64", False), + ("include_image_base64", True), + ("image_limit", 1), + ("image_min_size", 300), + ("confidence_scores_granularity", "page"), + ("confidence_scores_granularity", "word"), + ), +) +def test_vertex_mistral_strategy_reaches_every_2505_scalar(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI), + lambda candidate: field in candidate.model_fields_set and getattr(candidate, field) == value, + ) + + assert getattr(sdk_input, field) == value + + +@pytest.mark.parametrize( + "fields", + ( + frozenset({"bbox_annotation_format"}), + frozenset({"document_annotation_format"}), + frozenset({"document_annotation_format", "document_annotation_prompt"}), + ), +) +def test_vertex_mistral_strategy_reaches_every_2505_schema_group(fields: frozenset[str]) -> None: + sdk_input: Final = _find_fixture( + vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI), + lambda candidate: ( + frozenset(candidate.model_fields_set) - {"model", "document", "vertex_project", "vertex_location"} == fields + ), + ) + + assert frozenset(sdk_input.model_fields_set) - {"model", "document", "vertex_project", "vertex_location"} == fields + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI)) +def test_vertex_mistral_strategy_exercises_url_conversion_and_inline_bypass( + sdk_input: VertexMistralOcrSdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = { + name: value + for name, value in sdk_kwargs.items() + if name not in {"model", "document", "vertex_project", "vertex_location"} + } + config: Final = VertexAIOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + + request: Final = _transform_with_stubbed_download(transform_request, model, document, mapped) + + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + source: Final = document[source_key] + expected_document: Final = dict(document) + if not source.startswith("data:"): + media_type: Final = "image/png" if document["type"] == "image_url" else "application/pdf" + expected_document[source_key] = f"data:{media_type};base64,AA==" + + assert mapped == optional_params + assert cast(dict[str, object], request.data) == { + "model": model, + "document": expected_document, + **optional_params, + } + + +@settings(max_examples=30, deadline=None) +@given(sdk_input=vertex_deepseek_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI)) +def test_vertex_deepseek_strategy_only_generates_litellm_inputs( + sdk_input: VertexDeepSeekOcrSdkInput, +) -> None: + assert sdk_input.vertex_project == "project-1" + assert "contract" not in sdk_input.as_sdk_kwargs() + assert _document_transport(sdk_input.document) == ("image_url", "data") + + +def test_vertex_deepseek_strategy_reaches_documented_image_branch() -> None: + sdk_input: Final = _find_fixture( + vertex_deepseek_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI), + lambda candidate: _document_transport(candidate.document) == ("image_url", "data"), + ) + + assert _document_transport(sdk_input.document) == ("image_url", "data") diff --git a/tests/test_litellm/ocr/test_fixture_store.py b/tests/test_litellm/ocr/test_fixture_store.py new file mode 100644 index 00000000000..a01b6805e9a --- /dev/null +++ b/tests/test_litellm/ocr/test_fixture_store.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from queue import Queue +from typing import Final, Protocol, cast + +import pytest + +from tests.route_parity.fixtures.cassette import deserialize_cassette +from tests.route_parity.fixtures.pytest_support import parametrize_recorded_fixtures +from tests.route_parity.fixtures.store import FixtureEnvelope, read_fixture, recorded_fixtures +from tests.test_litellm.ocr.conftest import ocr_fixture_id, ocr_fixture_marks +from tests.test_litellm.ocr.fixtures.migrate import migrate_fixture +from tests.test_litellm.ocr.fixtures.models import OcrParityCase + + +class _Parameter(Protocol): + values: tuple[OcrParityCase, ...] + marks: tuple[pytest.Mark, ...] + + +@dataclass(frozen=True, slots=True) +class _MetafuncSpy: + fixturenames: tuple[str, ...] + calls: Queue[tuple[object, ...]] + + def parametrize(self, *args: object, **_kwargs: object) -> None: + self.calls.put(args) + + +def test_recorded_fixture_parametrization_applies_case_specific_marks() -> None: + calls: Final[Queue[tuple[object, ...]]] = Queue() + metafunc: Final = _MetafuncSpy(fixturenames=("ocr_fixture",), calls=calls) + + parametrize_recorded_fixtures( + cast(pytest.Metafunc, metafunc), + fixture_name="ocr_fixture", + case_type=OcrParityCase, + env_var="UNCONFIGURED_OCR_FIXTURE_TEST_DIRECTORY", + default_directory=Path(__file__).with_name("fixtures") / "data", + regeneration_command="unused", + id_builder=ocr_fixture_id, + marks_builder=ocr_fixture_marks, + ) + + parameters: Final = cast(tuple[_Parameter, ...], calls.get_nowait()[1]) + reducto_parameters: Final = tuple( + parameter + for parameter in parameters + if parameter.values[0].litellm_input.contract in {"reducto_v3", "reducto_legacy"} + ) + supported_parameters: Final = tuple(parameter for parameter in parameters if parameter not in reducto_parameters) + + assert reducto_parameters + assert supported_parameters + assert all(len(parameter.marks) == 1 for parameter in reducto_parameters) + assert all(parameter.marks[0].name == "xfail" for parameter in reducto_parameters) + assert all(parameter.marks[0].kwargs["strict"] is False for parameter in reducto_parameters) + assert all(parameter.marks == () for parameter in supported_parameters) + + +def test_legacy_fixture_migration_preserves_responses_and_labels_reconstructed_requests(tmp_path: Path) -> None: + case: Final = recorded_fixtures(Path(__file__).with_name("fixtures") / "data" / "mistral-ocr", OcrParityCase)[0] + timestamp: Final = datetime(2020, 1, 1, tzinfo=timezone.utc) + envelope: Final = FixtureEnvelope( + schema_version=1, + recorded_at=timestamp, + case=case.model_dump(mode="json", exclude_unset=True), + ) + legacy_path: Final = tmp_path / "legacy.json" + legacy_path.write_text(envelope.model_dump_json()) + + destination: Final = migrate_fixture(legacy_path) + + assert not legacy_path.exists() + assert read_fixture(destination, OcrParityCase) == case + cassette: Final = deserialize_cassette(destination.read_text()) + assert cassette.recorded_at == timestamp + assert cassette.parity.request_source == "python_replay" + assert len(cassette.interactions) == len(case.provider_responses) + assert cassette.interactions[0].request.method == "POST" + assert cassette.interactions[0].request.uri == "http://parity-provider.invalid/v1/ocr" + assert "authorization" not in cassette.interactions[0].request.headers diff --git a/tests/test_litellm/ocr/test_record_fixtures.py b/tests/test_litellm/ocr/test_record_fixtures.py new file mode 100644 index 00000000000..4ed4afcca38 --- /dev/null +++ b/tests/test_litellm/ocr/test_record_fixtures.py @@ -0,0 +1,366 @@ +from __future__ import annotations + +import queue +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final, cast + +import pytest +from hypothesis import find, settings +from hypothesis.strategies import SearchStrategy + +from tests.route_parity.fixtures.cli import parse_recording_args +from tests.route_parity.fixtures.inputs import generate_case_inputs +from tests.route_parity.fixtures.media import structured_pdf_data_uri +from tests.test_litellm.ocr.fixtures.azure import ( + AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS, + AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS, + AZURE_MISTRAL_MODELS, + AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS, +) +from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase +from tests.test_litellm.ocr.fixtures.common import OcrFixtureClient, OcrRecordingTarget +from tests.test_litellm.ocr.fixtures.mistral import MISTRAL_MODELS, MISTRAL_PROVIDER_REJECTED_INPUTS +from tests.test_litellm.ocr.fixtures.record import ( + discover_targets as discover_targets_with_media, +) +from tests.test_litellm.ocr.fixtures.record import ( + require_targets, +) +from tests.test_litellm.ocr.fixtures.reducto import ( + REDUCTO_LEGACY_MODELS, + REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS, + REDUCTO_V3_MODELS, + REDUCTO_V3_PROVIDER_REJECTED_INPUTS, +) +from tests.test_litellm.ocr.fixtures.vertex import ( + VERTEX_DEEPSEEK_MODELS, + VERTEX_MISTRAL_MODELS, + vertex_deepseek_provider_rejected_inputs, + vertex_mistral_provider_rejected_inputs, +) + + +class _UnusedOcrClient: + def execute(self, api_base: str, api_key: str, case_input: OcrSdkInputBase) -> None: + raise AssertionError(f"unexpected SDK call to {api_base} with {api_key!r} and {case_input!r}") + + +@dataclass(frozen=True, slots=True) +class _RecordingOcrClient: + calls: queue.SimpleQueue[dict[str, object]] + + def execute(self, api_base: str, api_key: str, case_input: OcrSdkInputBase) -> None: + self.calls.put({"api_base": api_base, "api_key": api_key, **case_input.as_sdk_kwargs()}) + + +_UNUSED_OCR_CLIENT: Final = _UnusedOcrClient() +_MISTRAL_PARAMS: Final = frozenset( + { + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "document_annotation_prompt", + "extract_header", + "extract_footer", + "table_format", + "confidence_scores_granularity", + "include_blocks", + } +) +_MISTRAL_2512_PARAMS: Final = _MISTRAL_PARAMS - {"include_blocks"} +_MISTRAL_2505_PARAMS: Final = _MISTRAL_2512_PARAMS - {"extract_header", "extract_footer", "table_format"} +_AZURE_MISTRAL_PARAMS: Final = _MISTRAL_2505_PARAMS - {"document_annotation_prompt"} +_FIND_SETTINGS: Final = settings(max_examples=2_000, deadline=None, derandomize=True, database=None) +_INLINE_IMAGE_DATA_URI: Final = "data:image/png;base64,dGVzdA==" + + +def discover_targets(environ: Mapping[str, str], client: OcrFixtureClient) -> tuple[OcrRecordingTarget, ...]: + return discover_targets_with_media(environ, client, _INLINE_IMAGE_DATA_URI) + + +def _model(case_input: OcrSdkInputBase) -> str: + model: Final = case_input.canonical_input().get("model") + assert isinstance(model, str) + return model + + +def _find_input( + strategy: SearchStrategy[OcrSdkInputBase], + predicate: Callable[[OcrSdkInputBase], bool], +) -> OcrSdkInputBase: + return find(strategy, predicate, settings=_FIND_SETTINGS) + + +def _document_transport(case_input: OcrSdkInputBase) -> tuple[str, str]: + document: Final = cast(dict[str, object], case_input.canonical_input()["document"]) + document_type: Final = cast(str, document["type"]) + source: Final = document["image_url"] if document_type == "image_url" else document["document_url"] + assert isinstance(source, str) + return document_type, "data" if source.startswith("data:") else "remote" + + +def test_parse_args_has_no_model_selection() -> None: + args: Final = parse_recording_args(["--examples", "2", "--concurrency", "3", "--fixture-dir", "/tmp/ocr"]) + + assert args.examples == 2 + assert args.concurrency == 3 + assert args.fixture_dir == Path("/tmp/ocr") + with pytest.raises(SystemExit): + parse_recording_args(["--model", "mistral/mistral-ocr-latest"]) + + +@pytest.mark.parametrize( + "environ", + ( + {}, + {"MISTRAL_API_KEY": ""}, + {"LITELLM_API_KEY": "generic-key"}, + ), +) +def test_discovery_requires_provider_specific_key(environ: dict[str, str]) -> None: + assert discover_targets(environ, _UNUSED_OCR_CLIENT) == () + + +def test_no_discovered_targets_has_actionable_error() -> None: + with pytest.raises(SystemExit, match="supported provider API key"): + require_targets(()) + + +def test_discovery_is_explicit_per_available_provider_boundary() -> None: + targets: Final = discover_targets( + { + "MISTRAL_API_KEY": "mistral-secret", + "REDUCTO_API_KEY": "reducto-secret", + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "AZURE_DOCUMENT_INTELLIGENCE_API_KEY": "document-secret", + "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT": "https://document.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + + assert tuple(target.name for target in targets) == ( + "mistral-ocr", + "azure-mistral", + "azure-document-intelligence", + "vertex-mistral", + "vertex-deepseek", + "reducto-v3", + "reducto-legacy", + ) + assert all("secret" not in repr(target) for target in targets) + + +def test_azure_mistral_discovery_enumerates_registered_models() -> None: + environ: Final = { + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + } + target: Final = discover_targets(environ, _UNUSED_OCR_CLIENT)[0] + + for model in AZURE_MISTRAL_MODELS: + assert ( + _model( + _find_input( + target.strategy, + lambda case_input, expected_model=model: _model(case_input) == expected_model, + ) + ) + == model + ) + + +@pytest.mark.parametrize( + ("configured", "expected"), + ( + (None, "https://api.mistral.ai"), + ("https://mistral.example/v1", "https://mistral.example"), + ("https://mistral.example/", "https://mistral.example"), + ), +) +def test_mistral_target_uses_canonical_model_and_normalized_base( + configured: str | None, + expected: str, +) -> None: + environ: Final = { + "MISTRAL_API_KEY": "mistral-secret", + **({"MISTRAL_API_BASE": configured} if configured is not None else {}), + } + targets: Final = discover_targets(environ, _UNUSED_OCR_CLIENT) + + assert len(targets) == 1 + target: Final = targets[0] + assert target.name == "mistral-ocr" + assert target.upstream.base_url == expected + assert "mistral-secret" not in repr(target) + case_inputs: Final = generate_case_inputs(target.strategy, examples=1) + assert len(case_inputs) == 1 + assert case_inputs[0].canonical_input()["model"] in MISTRAL_MODELS + + +def test_mistral_target_invocation_forwards_discovered_credentials() -> None: + calls: Final[queue.SimpleQueue[dict[str, object]]] = queue.SimpleQueue() + + client: Final = _RecordingOcrClient(calls) + target: Final = discover_targets({"MISTRAL_API_KEY": "mistral-secret"}, client)[0] + case_input: Final = generate_case_inputs(target.strategy, examples=1)[0] + + target.invocation.execute("http://127.0.0.1:1234", case_input) + + kwargs: Final = calls.get_nowait() + assert kwargs["api_base"] == "http://127.0.0.1:1234" + assert kwargs["api_key"] == "mistral-secret" + assert kwargs["model"] in MISTRAL_MODELS + + +def test_every_target_strategy_reaches_every_recording_model_and_coverage_param() -> None: + targets: Final = discover_targets( + { + "MISTRAL_API_KEY": "mistral-secret", + "REDUCTO_API_KEY": "reducto-secret", + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "AZURE_DOCUMENT_INTELLIGENCE_API_KEY": "document-secret", + "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT": "https://document.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + expected: Final[dict[str, tuple[tuple[str, ...], frozenset[str]]]] = { + "mistral-ocr": (MISTRAL_MODELS, _MISTRAL_PARAMS), + "azure-mistral": (AZURE_MISTRAL_MODELS, _AZURE_MISTRAL_PARAMS), + "azure-document-intelligence": ( + AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS, + frozenset({"pages", "features", "req_format"}), + ), + "vertex-mistral": (VERTEX_MISTRAL_MODELS, _MISTRAL_2505_PARAMS), + "vertex-deepseek": (VERTEX_DEEPSEEK_MODELS, frozenset[str]()), + "reducto-v3": (REDUCTO_V3_MODELS, frozenset({"formatting", "retrieval", "settings"})), + "reducto-legacy": (REDUCTO_LEGACY_MODELS, frozenset[str]()), + } + + for target in targets: + expected_models, expected_params = expected[target.name] + for model in expected_models: + assert ( + _model( + _find_input( + target.strategy, + lambda case_input, expected_model=model: _model(case_input) == expected_model, + ) + ) + == model + ) + for param in expected_params: + reached = _find_input( + target.strategy, + lambda case_input, expected_param=param: expected_param in case_input.as_sdk_kwargs(), + ) + assert param in reached.as_sdk_kwargs() + document = cast(dict[str, object], reached.canonical_input()["document"]) + assert document == {"type": "document_url", "document_url": structured_pdf_data_uri()} + + +@pytest.mark.parametrize("target_name", ("mistral-ocr", "azure-mistral", "vertex-mistral")) +def test_mistral_recording_targets_reach_every_transport_branch(target_name: str) -> None: + targets: Final = discover_targets( + { + "MISTRAL_API_KEY": "mistral-secret", + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + target: Final = next(candidate for candidate in targets if candidate.name == target_name) + + for transport in ( + ("image_url", "remote"), + ("image_url", "data"), + ("document_url", "remote"), + ("document_url", "data"), + ): + reached = _find_input( + target.strategy, + lambda case_input, expected=transport: _document_transport(case_input) == expected, + ) + assert _document_transport(reached) == transport + + +def test_vertex_deepseek_recording_reaches_documented_image_branch() -> None: + targets: Final = discover_targets( + { + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + target: Final = next(candidate for candidate in targets if candidate.name == "vertex-deepseek") + case_input: Final = _find_input( + target.strategy, + lambda candidate: _document_transport(candidate) == ("image_url", "data"), + ) + + assert _document_transport(case_input) == ("image_url", "data") + + +def test_only_intentional_provider_failures_are_fixed_inputs() -> None: + targets: Final = discover_targets( + { + "MISTRAL_API_KEY": "mistral-secret", + "REDUCTO_API_KEY": "reducto-secret", + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "AZURE_DOCUMENT_INTELLIGENCE_API_KEY": "document-secret", + "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT": "https://document.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + + expected: Final[dict[str, tuple[OcrSdkInputBase, ...]]] = { + "mistral-ocr": MISTRAL_PROVIDER_REJECTED_INPUTS, + "azure-mistral": AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS, + "azure-document-intelligence": AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS, + "vertex-mistral": vertex_mistral_provider_rejected_inputs("project-1", "us-central1", _INLINE_IMAGE_DATA_URI), + "vertex-deepseek": vertex_deepseek_provider_rejected_inputs("project-1", "us-central1", _INLINE_IMAGE_DATA_URI), + "reducto-v3": REDUCTO_V3_PROVIDER_REJECTED_INPUTS, + "reducto-legacy": REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS, + } + + assert {target.name for target in targets} == expected.keys() + for target in targets: + assert target.required_inputs == expected[target.name] + generated: Final = generate_case_inputs(target.strategy, examples=20) + assert all(case_input not in target.required_inputs for case_input in generated) + + +def test_mistral_adapters_preserve_omitted_optional_params() -> None: + targets: Final = discover_targets( + { + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + + baselines: Final = tuple( + _find_input( + target.strategy, + lambda case_input: _MISTRAL_PARAMS.isdisjoint(case_input.as_sdk_kwargs()), + ) + for target in targets + ) + assert all(_MISTRAL_PARAMS.isdisjoint(baseline.as_sdk_kwargs()) for baseline in baselines) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index 395b151b224..565c92ec483 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -368,6 +368,18 @@ def test_load_rust_ocr_uses_compiled_extension(monkeypatch): def test_timeout_to_seconds_handles_float_timeout_and_none(): assert rust_bridge._timeout_to_seconds(12.5) == 12.5 assert rust_bridge._timeout_to_seconds(None) is None + + +def test_ocr_provider_error_uses_resolved_request_url(): + error = rust_bridge._OcrProviderError( + 429, + "rate limited", + "https://example.azure.com/documentintelligence/documentModels/read:analyze", + ) + + assert str(error.response.request.url) == ( + "https://example.azure.com/documentintelligence/documentModels/read:analyze" + ) assert rust_bridge._timeout_to_seconds(httpx.Timeout(30.0, read=42.0)) == 42.0 @@ -662,18 +674,34 @@ def test_ocr_routes_to_rust_when_enabled(fake_bridge): assert call["optional_params"].get("include_image_base64") is True -def test_ocr_routes_azure_ai_to_rust_when_enabled(fake_bridge): +@pytest.mark.parametrize( + ("model", "provider"), + ( + ("azure_ai/pixtral-12b-2409", "azure_ai"), + ("vertex_ai/mistral-ocr-2505", "vertex_ai"), + ), +) +def test_ocr_routes_supported_provider_to_rust( + fake_bridge: RecordingBridge, + model: str, + provider: str, +) -> None: + provider_kwargs: dict[str, object] = ( + {"vertex_project": "project-1", "vertex_location": "us-central1"} if provider == "vertex_ai" else {} + ) response = litellm.ocr( - model="azure_ai/pixtral-12b-2409", + model=model, document=DOCUMENT, api_key="sk-test", - api_base="https://example.services.ai.azure.com", + api_base="https://example.com", + **provider_kwargs, ) assert isinstance(response, OCRResponse) assert len(fake_bridge.calls) == 1 - assert fake_bridge.calls[0]["model"] == "pixtral-12b-2409" - assert fake_bridge.calls[0]["custom_llm_provider"] == "azure_ai" + call = fake_bridge.calls[0] + assert call["model"] == model.rsplit("/", 1)[-1] + assert call["custom_llm_provider"] == provider def test_ocr_rust_path_converts_file_document_before_bridge(fake_bridge): @@ -822,6 +850,27 @@ def test_ocr_unsupported_provider_skips_rust(monkeypatch): assert bridge.calls == [] +def test_ocr_non_string_header_uses_python_path(monkeypatch): + bridge = RecordingBridge() + litellm.use_litellm_rust(True, ocr=bridge) + + def fake_handler_ocr(**kwargs): + assert kwargs["headers"] == {"x-invalid": 1} + return OCRResponse(pages=[], model="mistral-ocr-latest", object="ocr") + + monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", fake_handler_ocr) + + response = litellm.ocr( + model=MODEL, + document=DOCUMENT, + api_key="sk-test", + extra_headers={"x-invalid": 1}, + ) + + assert isinstance(response, OCRResponse) + assert bridge.calls == [] + + def test_ocr_provider_configs_expose_api_key_env_vars(): from litellm.llms.azure_ai.ocr.document_intelligence.transformation import ( AzureDocumentIntelligenceOCRConfig, diff --git a/tests/test_litellm/ocr/test_sdk_parity.py b/tests/test_litellm/ocr/test_sdk_parity.py new file mode 100644 index 00000000000..8aa5713a63c --- /dev/null +++ b/tests/test_litellm/ocr/test_sdk_parity.py @@ -0,0 +1,463 @@ +from __future__ import annotations + +import asyncio +import sys +import traceback +from collections.abc import Awaitable, Callable, Coroutine, Generator +from contextlib import contextmanager +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Final, cast + +import pytest + +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.rust_bridge import get_native_bridge +from litellm.rust_bridge import ocr as rust_ocr_bridge +from litellm.rust_bridge.ocr import RustAocr, RustOcr +from tests.route_parity.compare import assert_model_parity, assert_parity, assert_request_parity +from tests.route_parity.fixtures.store import recorded_fixtures +from tests.route_parity.inprocess import run_in_process +from tests.route_parity.models import ( + SDKCommand, + SDKError, + SDKReport, + SDKSuccess, + WorkerFailure, + WorkerResult, + WorkerSuccess, + sdk_error_report, +) +from tests.route_parity.replay import replay_server +from tests.route_parity.runner import ( + ExecutionVariant, + SubprocessRunner, + SubprocessWorker, + execution_worker_pair, + parity_worker_main, + run_execution, +) +from tests.test_litellm.ocr.fixtures.config import configured_fixture_directory +from tests.test_litellm.ocr.fixtures.models import OcrParityCase, OcrSdkInput + +API_KEY: Final = "test-key" +PYTHON_HTTP_SENTINEL: Final = "python-ocr-parity-fallback" +PYTHON_VARIANT: Final = ExecutionVariant(name="Python", environment=(("LITELLM_USE_RUST_OCR", "0"),)) +RUST_VARIANT: Final = ExecutionVariant(name="Rust", environment=(("LITELLM_USE_RUST_OCR", "1"),)) + + +class SDKRoute(str, Enum): + OCR = "ocr" + AOCR = "aocr" + + +@dataclass(frozen=True, slots=True) +class InvalidOcrCase: + name: str + model: str + document: object + expected_exception_type: str + expected_status_code: int + expected_message: str + extra_kwargs: tuple[tuple[str, object], ...] = () + expected_rust_calls: int = 0 + + +INVALID_OCR_CASES: Final = ( + InvalidOcrCase( + name="unsupported_provider", + model="openai/gpt-4o", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="OCR is not supported for provider: openai", + ), + InvalidOcrCase( + name="unsupported_reducto_model", + model="reducto/parse-v4", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="OCR is not supported for provider: reducto", + ), + InvalidOcrCase( + name="unknown_provider_prefix", + model="not_a_provider/model", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, + expected_message="LLM Provider NOT provided", + ), + InvalidOcrCase( + name="non_object_document", + model="mistral/mistral-ocr-latest", + document=[], + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="document must be a dict", + ), + InvalidOcrCase( + name="missing_document_type", + model="mistral/mistral-ocr-latest", + document={}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Invalid document type: None", + ), + InvalidOcrCase( + name="unsupported_document_type", + model="mistral/mistral-ocr-latest", + document={"type": "text", "text": "not a document"}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Invalid document type: text", + ), + InvalidOcrCase( + name="missing_document_url", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "document_url"}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Document URL is required", + expected_rust_calls=1, + ), + InvalidOcrCase( + name="missing_image_url", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "image_url"}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Document URL is required", + expected_rust_calls=1, + ), + InvalidOcrCase( + name="invalid_request_format", + model="mistral/mistral-ocr-latest", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.UnsupportedParamsError", + expected_status_code=400, + expected_message="Invalid `req_format`: 'bogus'", + extra_kwargs=(("req_format", "bogus"),), + ), + InvalidOcrCase( + name="invalid_document_intelligence_pages", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="`pages` integers must be >= 0", + extra_kwargs=(("pages", [-1]),), + ), + InvalidOcrCase( + name="invalid_document_intelligence_features", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Invalid `features` for Azure Document Intelligence", + extra_kwargs=(("features", [1]),), + ), +) + + +def _call_kwargs(sdk_input: OcrSdkInput, mock_url: str, route: SDKRoute) -> dict[str, object]: + return { + **sdk_input.as_sdk_kwargs(), + "api_base": mock_url, + "api_key": API_KEY, + "extra_headers": {"x-litellm-parity-route": route.value}, + } + + +def _execute_sdk_call( + call_kwargs: dict[str, object], + route: SDKRoute, + event_loop: asyncio.AbstractEventLoop, +) -> SDKReport: + import litellm + + try: + if route is SDKRoute.OCR: + sync_route: Final = cast(Callable[..., OCRResponse], litellm.ocr) + response: Final = sync_route(**call_kwargs) + return SDKSuccess(response=response.model_dump(mode="json")) + async_route: Final = cast(Callable[..., Coroutine[object, object, OCRResponse]], litellm.aocr) + async_response: Final = event_loop.run_until_complete(async_route(**call_kwargs)) + return SDKSuccess(response=async_response.model_dump(mode="json")) + except Exception as error: + return sdk_error_report(error) + + +def _execute_sdk_case( + sdk_input: OcrSdkInput, + route: SDKRoute, + mock_url: str, + event_loop: asyncio.AbstractEventLoop, +) -> SDKReport: + call_kwargs: Final = _call_kwargs(sdk_input, mock_url, route) + return _execute_sdk_call(call_kwargs, route, event_loop) + + +def _execute_recorded_sdk_case( + sdk_input: OcrSdkInput, + route: SDKRoute, + mock_url: str, + event_loop: asyncio.AbstractEventLoop, +) -> OCRResponse | SDKError: + import litellm + + call_kwargs: Final = _call_kwargs(sdk_input, mock_url, route) + try: + if route is SDKRoute.OCR: + sync_route: Final = cast(Callable[..., OCRResponse], litellm.ocr) + return sync_route(**call_kwargs) + async_route: Final = cast(Callable[..., Coroutine[object, object, OCRResponse]], litellm.aocr) + return event_loop.run_until_complete(async_route(**call_kwargs)) + except Exception as error: + return sdk_error_report(error) + + +def _execute_invalid_sdk_case( + case: InvalidOcrCase, + route: SDKRoute, + mock_url: str, + event_loop: asyncio.AbstractEventLoop, +) -> SDKReport: + call_kwargs: Final = { + "model": case.model, + "document": case.document, + "api_base": mock_url, + "api_key": API_KEY, + "extra_headers": {"x-litellm-parity-route": route.value}, + **dict(case.extra_kwargs), + } + return _execute_sdk_call(call_kwargs, route, event_loop) + + +class _RustOcrSpy: + def __init__(self, delegate: RustOcr) -> None: + self.delegate: Final = delegate + self.calls = 0 + + def __call__( + self, + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str | None, + extra_headers: dict[str, object] | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> dict[str, object]: + self.calls += 1 + return self.delegate( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + optional_params=optional_params, + timeout_seconds=timeout_seconds, + ) + + +class _RustAocrSpy: + def __init__(self, delegate: RustAocr) -> None: + self.delegate: Final = delegate + self.calls = 0 + + async def __call__( + self, + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str | None, + extra_headers: dict[str, object] | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> dict[str, object]: + self.calls += 1 + result: Final[Awaitable[dict[str, object]]] = self.delegate( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + optional_params=optional_params, + timeout_seconds=timeout_seconds, + ) + return await result + + +@contextmanager +def _restore_rust_ocr_state() -> Generator[None]: + from litellm.rust_bridge import configuration + + config: Final = configuration._CONFIGURATION # pyright: ignore[reportPrivateUsage] # restore test state + sync_binding: Final = rust_ocr_bridge._OCR # pyright: ignore[reportPrivateUsage] # restore test state + async_binding: Final = rust_ocr_bridge._AOCR # pyright: ignore[reportPrivateUsage] # restore test state + enabled: Final = config.override + ocr_impl: Final = sync_binding._override # pyright: ignore[reportPrivateUsage] # preserve unset binding + aocr_impl: Final = async_binding._override # pyright: ignore[reportPrivateUsage] # preserve unset binding + try: + yield + finally: + config.override = enabled + sync_binding._override = ocr_impl # pyright: ignore[reportPrivateUsage] # restore exact binding state + async_binding._override = aocr_impl # pyright: ignore[reportPrivateUsage] # restore exact binding state + + +def _native_spies() -> tuple[_RustOcrSpy, _RustAocrSpy]: + native_bridge: Final = get_native_bridge() + if native_bridge is None: + pytest.fail("native Rust bridge is required for OCR parity testing") + sync_spy: Final = _RustOcrSpy(cast(RustOcr, getattr(native_bridge, "ocr"))) + async_spy: Final = _RustAocrSpy(cast(RustAocr, getattr(native_bridge, "aocr"))) + return sync_spy, async_spy + + +@pytest.fixture(scope="module") +def sdk_workers() -> Generator[tuple[SubprocessWorker, SubprocessWorker]]: + runner: Final = SubprocessRunner( + entrypoint=Path(__file__), + baseline_user_agent=PYTHON_HTTP_SENTINEL, + route_label="OCR", + ) + with execution_worker_pair(runner, PYTHON_VARIANT, RUST_VARIANT) as workers: + yield workers + + +@pytest.fixture(scope="module") +def startup_ocr_fixture() -> OcrParityCase: + directory: Final = configured_fixture_directory() + fixtures: Final = recorded_fixtures(directory, OcrParityCase) + if not fixtures: + pytest.skip(f"no recorded fixtures in {directory}") + return fixtures[0] + + +@pytest.mark.parametrize("route", tuple(SDKRoute), ids=tuple(route.value for route in SDKRoute)) +def test_recorded_ocr_sdk_parity( + ocr_fixture: OcrParityCase, + route: SDKRoute, +) -> None: + sync_spy, async_spy = _native_spies() + event_loop: Final = asyncio.new_event_loop() + try: + with _restore_rust_ocr_state(), replay_server() as provider: + rust_ocr_bridge.set_rust_ocr(ocr=sync_spy, aocr=async_spy) + rust_ocr_bridge.use_litellm_rust(False) + python: Final = run_in_process( + provider, + ocr_fixture.provider_responses, + lambda mock_url: _execute_recorded_sdk_case(ocr_fixture.litellm_input, route, mock_url, event_loop), + ) + assert sync_spy.calls == 0 + assert async_spy.calls == 0 + + rust_ocr_bridge.use_litellm_rust(True) + rust: Final = run_in_process( + provider, + ocr_fixture.provider_responses, + lambda mock_url: _execute_recorded_sdk_case(ocr_fixture.litellm_input, route, mock_url, event_loop), + ) + finally: + event_loop.close() + + assert sync_spy.calls == (1 if route is SDKRoute.OCR else 0) + assert async_spy.calls == (1 if route is SDKRoute.AOCR else 0) + assert_request_parity(python.requests, rust.requests) + if any(response.status_code >= 400 for response in ocr_fixture.provider_responses): + assert isinstance(python.response, SDKError) + if isinstance(python.response, SDKError): + assert python.response == rust.response + else: + assert isinstance(rust.response, OCRResponse) + assert_model_parity(python.response, rust.response) + + +@pytest.mark.parametrize("case", INVALID_OCR_CASES, ids=tuple(case.name for case in INVALID_OCR_CASES)) +@pytest.mark.parametrize("route", tuple(SDKRoute), ids=tuple(route.value for route in SDKRoute)) +def test_invalid_ocr_sdk_parity(case: InvalidOcrCase, route: SDKRoute) -> None: + sync_spy, async_spy = _native_spies() + event_loop: Final = asyncio.new_event_loop() + try: + with _restore_rust_ocr_state(), replay_server() as provider: + rust_ocr_bridge.set_rust_ocr(ocr=sync_spy, aocr=async_spy) + rust_ocr_bridge.use_litellm_rust(False) + python: Final = run_in_process( + provider, + (), + lambda mock_url: _execute_invalid_sdk_case(case, route, mock_url, event_loop), + ) + assert sync_spy.calls == 0 + assert async_spy.calls == 0 + + rust_ocr_bridge.use_litellm_rust(True) + rust: Final = run_in_process( + provider, + (), + lambda mock_url: _execute_invalid_sdk_case(case, route, mock_url, event_loop), + ) + finally: + event_loop.close() + + assert sync_spy.calls == (case.expected_rust_calls if route is SDKRoute.OCR else 0) + assert async_spy.calls == (case.expected_rust_calls if route is SDKRoute.AOCR else 0) + assert python.requests == () + assert rust.requests == () + assert python.response == rust.response + assert isinstance(python.response, SDKError) + assert python.response.exception_type == case.expected_exception_type + assert python.response.status_code == case.expected_status_code + assert case.expected_message in python.response.message + + +def test_ocr_subprocess_startup_smoke( + startup_ocr_fixture: OcrParityCase, + tmp_path: Path, + sdk_workers: tuple[SubprocessWorker, SubprocessWorker], +) -> None: + case_file: Final = tmp_path / "ocr-startup-smoke.json" + case_file.write_text(startup_ocr_fixture.model_dump_json(indent=2, exclude_unset=True), encoding="utf-8") + python_worker, rust_worker = sdk_workers + python: Final = run_execution( + python_worker, + case_file, + SDKRoute.OCR.value, + startup_ocr_fixture.provider_responses, + ) + rust: Final = run_execution( + rust_worker, + case_file, + SDKRoute.OCR.value, + startup_ocr_fixture.provider_responses, + ) + + assert_parity(python, rust, PYTHON_HTTP_SENTINEL) + + +def _execute_worker_command( + command_json: str, + mock_url: str, + event_loop: asyncio.AbstractEventLoop, +) -> WorkerResult: + try: + command: Final = SDKCommand.model_validate_json(command_json) + case_file: Final = Path(command.case_file) + route: Final = SDKRoute(command.route) + case: Final = OcrParityCase.model_validate_json(case_file.read_text(encoding="utf-8")) + return WorkerSuccess(report=_execute_sdk_case(case.litellm_input, route, mock_url, event_loop)) + except Exception: + return WorkerFailure(error=traceback.format_exc()) + + +if __name__ == "__main__": + if len(sys.argv) != 3 or sys.argv[1] != "--parity-worker": + raise SystemExit("usage: test_sdk_parity.py --parity-worker MOCK_URL") + parity_worker_main(_execute_worker_command, sys.argv[2]) diff --git a/tests/test_litellm/proxy/common_utils/test_user_api_key_cache.py b/tests/test_litellm/proxy/common_utils/test_user_api_key_cache.py index 33e0d8bf38e..d1f979882ae 100644 --- a/tests/test_litellm/proxy/common_utils/test_user_api_key_cache.py +++ b/tests/test_litellm/proxy/common_utils/test_user_api_key_cache.py @@ -41,7 +41,7 @@ class FakeRedisCache(RedisCache): Records the ``ttl`` kwarg DualCache forwards on each Redis write for tests. """ - def __init__(self): # noqa: super().__init__ skipped intentionally + def __init__(self): self._store: dict[str, str] = {} self.last_ttl: Any = None diff --git a/tests/test_litellm/rust_bridge/test_runtime.py b/tests/test_litellm/rust_bridge/test_runtime.py index 9d0fd6d2098..27f0bd60c3a 100644 --- a/tests/test_litellm/rust_bridge/test_runtime.py +++ b/tests/test_litellm/rust_bridge/test_runtime.py @@ -2,6 +2,7 @@ from __future__ import annotations from types import SimpleNamespace +import httpx import pytest from litellm.exceptions import APIError @@ -120,3 +121,59 @@ def test_required_mode_rejects_unavailable_bridge() -> None: mode=runtime.FallbackMode.RUST_REQUIRED, context=context(), ) + + +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +@pytest.mark.asyncio +async def test_upstream_error_adapter_preserves_response_without_fallback(asynchronous: bool) -> None: + request = httpx.Request("POST", "https://example.com/ocr") + + def provider_error(status: int, message: str) -> Exception: + return httpx.HTTPStatusError( + message, + request=request, + response=httpx.Response(status, request=request), + ) + + error_context = runtime.BridgeErrorContext( + route="ocr", provider="mistral", model="model", upstream_error=provider_error + ) + + def fail() -> str: + raise RustUpstreamError(429, '{"message":"rate limited"}') + + async def afail() -> str: + return fail() + + def fallback() -> str: + pytest.fail("provider failure must not execute Python fallback") + + async def afallback() -> str: + return fallback() + + async def invoke() -> None: + if asynchronous: + await runtime.ainvoke( + native_call=afail, + fallback=afallback, + adapt=runtime.identity, + mode=runtime.FallbackMode.PYTHON, + context=error_context, + ) + else: + runtime.invoke( + native_call=fail, + fallback=fallback, + adapt=runtime.identity, + mode=runtime.FallbackMode.PYTHON, + context=error_context, + ) + + with pytest.raises(httpx.HTTPStatusError) as caught: + await invoke() + + assert str(caught.value) == '{"message":"rate limited"}' + assert caught.value.response.status_code == 429 + assert caught.value.request is request + assert isinstance(caught.value.__cause__, RustUpstreamError) + assert caught.value.headers == {"x-litellm-core": "rust", "x-litellm-rust": "true"} diff --git a/uv.lock b/uv.lock index aa59ff7b229..3996c9b472c 100644 --- a/uv.lock +++ b/uv.lock @@ -3373,6 +3373,98 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, ] +[[package]] +name = "hypothesis" +version = "6.165.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/e2/0fad246d2b6330e1f78479bfc566b5c22be82aee8a865cde9a08f648487d/hypothesis-6.165.10.tar.gz", hash = "sha256:68b45e09834cd80523cb1eb274463073c7a9af4e4ef7cff34d9615f355572d32", size = 503703, upload-time = "2026-08-16T22:56:15.404Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/c1/9a9538e6d185baf5cc7f15bc3b76e08efbb3de4b3c782f234356449c0dd7/hypothesis-6.165.10-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f839d29d0cc12048cf073d88ca4fdf94d420bc2b8afd69641ff6d496422ccd4f", size = 783243, upload-time = "2026-08-16T22:55:44.058Z" }, + { url = "https://files.pythonhosted.org/packages/a1/30/b70d9d79e871a75cbdeccd9067f20ecdb9eb2a1dfa03c630be3ad13b8b30/hypothesis-6.165.10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e10858f57ed0e74baa04393845f469fe8ad502c16ece4499bef7700c575611bd", size = 778815, upload-time = "2026-08-16T22:55:46.948Z" }, + { url = "https://files.pythonhosted.org/packages/db/52/6f0a9b7aab24b0635e2238f3fbddea5b54b17879ac813df42a3cc3384c5c/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76a7be86d986223b9f1bdb7e7cbcdb048649901fdb956c598ef73bdab1786cd5", size = 1108009, upload-time = "2026-08-16T22:54:53.082Z" }, + { url = "https://files.pythonhosted.org/packages/f6/06/8d0d4e11ff02350d09ec9f9e90af354158e59e16a8907ba5199a4ff2d7e8/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:717aea574e0e5edba2868aa66b1caae335d8f1ad3fb29f01dd6502953fa823a1", size = 1136596, upload-time = "2026-08-16T22:54:54.443Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/01a1e440f2e38dc1ccf5d597af5b8a0bee5f21b674c99c123b5554de9690/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4334058033e0214475f019e15492a50f3854fe8728cf51fe25c6191a2c3f8e52", size = 1135234, upload-time = "2026-08-16T22:55:08.911Z" }, + { url = "https://files.pythonhosted.org/packages/7d/18/8a26c24d3d9db20265f39df341ab265858c094e209571e3179cf237935f4/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2abb50cf1cf77d721de0a24c3f99d9c4ffdeb2cbd1e12aebb5a7a93e2b6b6d1f", size = 1157528, upload-time = "2026-08-16T22:56:02.159Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8e/ce3c829b1937402d7944420ca26a05a0c8563e894dcff03d34ffa279d306/hypothesis-6.165.10-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:3de69aa8b924b400291a3cc42aaf78e6ab65c905a3e7e1a5dc39d95ef1b428cb", size = 1112870, upload-time = "2026-08-16T22:54:55.919Z" }, + { url = "https://files.pythonhosted.org/packages/f2/1b/4c4926d6c9a2b5d7cc090cc1e91219d6796102aa2a2c4b8f961c939e60b5/hypothesis-6.165.10-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5841331c504e02d7c334591681cb8587cdd59dee7e149db6d3db8e3f9e9f02eb", size = 1149683, upload-time = "2026-08-16T22:55:30.567Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f9/df24eb28412f82465e2b7707f0ff1ec274d580bce389d4d9156617dc7bba/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2d0e0f8263d34dd8fa3b39eaa9a50bba56a8470b3dd9ebf6672d10840abe063e", size = 1283402, upload-time = "2026-08-16T22:54:18.054Z" }, + { url = "https://files.pythonhosted.org/packages/4d/07/c2b2a761300cf60b90ccebba4328175331e67d34f4fbd39429a7ddcdce49/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0c4e6869817c3cfdf5a2b4d348497b95159bdecb3365be732c9b8570e36a4eef", size = 1409948, upload-time = "2026-08-16T22:54:22.343Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ec/1c2bf1acdd0e273d81f833f85caf0ae5423db68a783554992fca36e6c541/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9f07ae36c3b093e13687a894e79fe69e98a94c0b67fef656c575247682218143", size = 1265023, upload-time = "2026-08-16T22:54:41.402Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a8/7f984908b7391160c7801b84e51ca8e4ba88c89e8d8811aa1aa7c03de73c/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:aff1f584c9538e8979cd180b1d70bf99bc16be19d4666414f49e5942b21a4f2c", size = 1282698, upload-time = "2026-08-16T22:56:06.998Z" }, + { url = "https://files.pythonhosted.org/packages/48/78/3a5d91c2d0250521736c42dfa2402b75049bc5fe2fb716c10bc84bb91ed1/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1f2c4db25fb8ec1a16a8dba580666337b8ffb1887c4cf1750cc954313897cef7", size = 1324816, upload-time = "2026-08-16T22:54:46.675Z" }, + { url = "https://files.pythonhosted.org/packages/6f/99/27450763853a034bca1574d3e0a315164b33ff49c3862df6872dda45e25e/hypothesis-6.165.10-cp310-abi3-win32.whl", hash = "sha256:b33dc30170a7402e03c180f2c5ef69dc077152f35b91621e9cebcde9c7d71746", size = 669039, upload-time = "2026-08-16T22:55:11.962Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fc/ff2988b72b5705ad9ca500444bf3f43e3c2f41edfa034bbfeb23b215791a/hypothesis-6.165.10-cp310-abi3-win_amd64.whl", hash = "sha256:e9f924aa610c0618445e1e8738c822c3190ce2a2699a0cb48ec3a351a96761f2", size = 675213, upload-time = "2026-08-16T22:55:01.697Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8b/821810d36f78d9d9421cd2c5d9d36983b45bb3575c3086276cc5c76f9f73/hypothesis-6.165.10-cp310-abi3-win_arm64.whl", hash = "sha256:1d305448e9bd8e2f4f3cea0eafd809efdaab4e998a0019bc615650c8463e42f1", size = 673537, upload-time = "2026-08-16T22:54:47.898Z" }, + { url = "https://files.pythonhosted.org/packages/26/61/5e89268ce03317fb9f82449a1b3efd9e599dee090288fd0cf7586c532fb1/hypothesis-6.165.10-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:73e6df02a6a62f8045b511c272f894d08e56d174504c793c9effcbc6778051a8", size = 783959, upload-time = "2026-08-16T22:55:29.078Z" }, + { url = "https://files.pythonhosted.org/packages/e1/e9/f4e0832e81bb53b70cf1712e28c867db64245b32595b594217452e7dbd8d/hypothesis-6.165.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8b20f44773a9ab84400465e318712d8c2ca16418d35b9f80aa27fdf2d690ad10", size = 779684, upload-time = "2026-08-16T22:54:57.698Z" }, + { url = "https://files.pythonhosted.org/packages/77/de/ea072d3359d5678771bed407f80439e8ac7ca905d1031b0372f61bf5746e/hypothesis-6.165.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb8c7d05ea27a093a92b250904095d71d924b6b44e5795a415c1b20c265f0c65", size = 1108540, upload-time = "2026-08-16T22:55:03.282Z" }, + { url = "https://files.pythonhosted.org/packages/28/56/e7c395cdaa3d6c28b944c1c3c516dee50d2b7b3aeafa31874b57009ca51f/hypothesis-6.165.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4dafd6d6ababfa3b14dd6e5f0378cb7c7d291895a31a40abcbb7cc74f396131", size = 1158089, upload-time = "2026-08-16T22:54:36.205Z" }, + { url = "https://files.pythonhosted.org/packages/b1/49/1c6d2c465b9c5fc3213f1be89be95ba53819ca0130248c484129ccfefb71/hypothesis-6.165.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fa74636a49fc8077413ce8db3e85f1c4aff880788bb55bda56253118e036fe5b", size = 1284125, upload-time = "2026-08-16T22:54:37.727Z" }, + { url = "https://files.pythonhosted.org/packages/7d/bc/7caf5ac3d0173bd57bd2a5ab854ca49a3664a4309257be1452f81025cc24/hypothesis-6.165.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2b112768cfb67f2b683e53e58c1a33d27811aacf60c942b8eb74635e469a73f6", size = 1325082, upload-time = "2026-08-16T22:54:38.952Z" }, + { url = "https://files.pythonhosted.org/packages/70/99/9d844330f570d6a4f127a683eab1e78c8263e6e72b16189f3534fa6bf6de/hypothesis-6.165.10-cp310-cp310-win_amd64.whl", hash = "sha256:56cb8c9055e50545fe6e3e5a560ec25a724673b2e4051f3c24d44e3ebc35dd72", size = 675082, upload-time = "2026-08-16T22:54:29.872Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c2/b9546ace11f241c9c02d389f258cb80c14447a8c885771c9f1f0bc1d85ca/hypothesis-6.165.10-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:592107a0faf6c9c3a63a8dbf13dfb1cbda1cf599b0bc11c953221b00204b9ce1", size = 783716, upload-time = "2026-08-16T22:55:36.624Z" }, + { url = "https://files.pythonhosted.org/packages/37/10/27c2fdd574fd798caf5e91eb51f7834b098f5d840ce733efb3fba79ef86e/hypothesis-6.165.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9180c362bde06fd05380298ded4e234fbc0d6ede0a864835bfd91c1e24283d5", size = 779507, upload-time = "2026-08-16T22:55:07.633Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b6/70bc23695f3783c4b0486b6cad47b08a20f791db4a3c1b25250add9659fa/hypothesis-6.165.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d623801ae3dcd97b77b983400ef3d48bf976648e4efff19929175322eaae074d", size = 1108406, upload-time = "2026-08-16T22:55:39.653Z" }, + { url = "https://files.pythonhosted.org/packages/71/4c/32e200bd7a352af4b7f4e3729aaa4cd002cb5fe8c4c6aef5599d0019f152/hypothesis-6.165.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20f6236cfb90b7817bb1a6a087589ca4aa46d73170f0dd62963952ed5dadc589", size = 1157850, upload-time = "2026-08-16T22:55:24.394Z" }, + { url = "https://files.pythonhosted.org/packages/03/a5/8efc2a9a484822efc0d0da466f50094e0f2c068187faaf33831fc905873e/hypothesis-6.165.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad0764730e8e3421601c2cc7e1f054a9206c60ea0917165d8d9193dc453f34f1", size = 1283704, upload-time = "2026-08-16T22:54:27.279Z" }, + { url = "https://files.pythonhosted.org/packages/46/2a/90cc8d7463929c04786f29600de45f3227c12fa9bed1d5b7ce319b05e1c9/hypothesis-6.165.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:10d9a650a4666b0914831f769703d36140ed8039fd19bf9b71f615b8541eccf2", size = 1325077, upload-time = "2026-08-16T22:55:16.561Z" }, + { url = "https://files.pythonhosted.org/packages/82/ac/bc16faba4b42883e3d290bfaceff51e258b63fbbdf789bf9fe88df1ce537/hypothesis-6.165.10-cp311-cp311-win_amd64.whl", hash = "sha256:5671d2b2bf83bd4b6f02e55b32d432506eff5358c82f39b460a849ce19a2666e", size = 674920, upload-time = "2026-08-16T22:55:42.613Z" }, + { url = "https://files.pythonhosted.org/packages/e9/45/cde4f78afe2b9e29caecf38319eedc1deb76aebcacbdd128e03cbb2511c3/hypothesis-6.165.10-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:637445c1593a2a9d1024fda50082f07bb56baedda78d90a25f64b8111727ef94", size = 784835, upload-time = "2026-08-16T22:54:45.429Z" }, + { url = "https://files.pythonhosted.org/packages/7f/81/847f30b81cbfd07607296b3ce43067cf4f80799bd9244167f587de9c8081/hypothesis-6.165.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:713f4ce4e82c26b53031f139de959bc9e8b54d3995aa824b89bbdf8229df2a45", size = 776419, upload-time = "2026-08-16T22:55:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/04/66/4c71c5be7a49d84b8c3a9278c1807c4c81181ab5474beb27df9d4c40dc0e/hypothesis-6.165.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9ff356e97e3ab09db07c8b675efa67340103874a0bae7465acb83dad7a35f7f", size = 1106830, upload-time = "2026-08-16T22:55:10.389Z" }, + { url = "https://files.pythonhosted.org/packages/e3/c4/e2cbd2810e79f7a452a8ea9f6c6438ee718ce938d8cc12252cf0b36a81d3/hypothesis-6.165.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a380bc99aa3b035e6a95a2201bf792d4082a04ca75babcc21849c2d0914bb28", size = 1156952, upload-time = "2026-08-16T22:55:53.35Z" }, + { url = "https://files.pythonhosted.org/packages/a8/8b/794ced36864825492ac3712d5acab5a257b4601e6a9dc2ccdd3937198f87/hypothesis-6.165.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e9acb2c4d9cb532c3fedea74159f7b923c8c036328c9239b4049e7aa073bdd81", size = 1280780, upload-time = "2026-08-16T22:54:34.983Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2d/550525442cdbcc2daf1f9bdd8ba35bcbde63db7c7a22f2ef137fbb49df2f/hypothesis-6.165.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8660572b2d424bf5369ea8990985225f70bd1615b76ecd9c25588a3b9307009f", size = 1324130, upload-time = "2026-08-16T22:55:48.659Z" }, + { url = "https://files.pythonhosted.org/packages/74/59/6caf69dd5fe03499ada94c9cec016bffcc164511c6b93fe680f01209b9ff/hypothesis-6.165.10-cp312-cp312-win_amd64.whl", hash = "sha256:3376f2594763aef14faa519b0fb27cae7ce9eeaab4c69efa07777499110306c9", size = 672337, upload-time = "2026-08-16T22:54:49.11Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fb/c82c5bd92864ffcf319772fedc8c9bf2dbe4ca14baa0fee6e49e67b5ba1c/hypothesis-6.165.10-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9d77c3be7b429875036ad0f0597c6e5cc6bb17894a4da005e3807de64d2673ad", size = 784726, upload-time = "2026-08-16T22:54:32.371Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b9/3d7acd08506da85557e65147b7f3fca8c47684e33be90bee0acb523920db/hypothesis-6.165.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:490c56b830772b0eca3b4b2cecb3741a1ed26b1d7206a279e1525dbf0aa95ee4", size = 776375, upload-time = "2026-08-16T22:55:13.303Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/922e8b3f9a706dd89d440b9545d2c6231c65e74da1c1fee3ff36c251b9c4/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed68e27b8a61e57a3ccdc7c5a14499e00b54dfe223087204d5d40b3b5ef58b6d", size = 1106763, upload-time = "2026-08-16T22:55:06.129Z" }, + { url = "https://files.pythonhosted.org/packages/01/39/f5b9a5d390d4edd1ad472334493ac442963ebeb4daaa74ff4bdac6ef292f/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6caadcd1afb62630ff5c5ff353626eaa616553a5971295ad6dc2b19ca8a39620", size = 1156778, upload-time = "2026-08-16T22:54:33.824Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5f/5fbe1be4326337fd6acefe2d18ed44007ee1dc1f98fe5b3c0eb22942364d/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9145fe43ebb22e66672967c3fab411793b226ed776e4fe282271bca6ad3c0bb", size = 1280756, upload-time = "2026-08-16T22:55:54.834Z" }, + { url = "https://files.pythonhosted.org/packages/25/c0/cf6f9e1ef632a1a75694eed0db3a02e6fc75c367a363e94acee52f043c64/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:79900a9920a0b1d3a626c03a90ac6bf7042e78d46906a565b86a0dbe926f1d96", size = 1323889, upload-time = "2026-08-16T22:55:56.567Z" }, + { url = "https://files.pythonhosted.org/packages/cc/cc/662b94880f260b0a88de1fdcf60fc9984f6e2a796da549542adc10a7bc83/hypothesis-6.165.10-cp313-cp313-win_amd64.whl", hash = "sha256:c01dd04044c472e47193b54f68e84e08d6ebf4f29551885aa959b015f7cd9747", size = 672346, upload-time = "2026-08-16T22:56:03.792Z" }, + { url = "https://files.pythonhosted.org/packages/3f/77/55e020c9c576532ff7d20bf8b1dfa052ecbd5ada1949b02f76c44c966f7e/hypothesis-6.165.10-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9ccac776b2ca93b324806facd526ccb45da0fd035001c899a35b02c44431e209", size = 784833, upload-time = "2026-08-16T22:55:21.255Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f2/01da2adf829cf549eaddcabb8e8072077fb3d26da4275f4c1e89b2c0af74/hypothesis-6.165.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e5f95f7b622e4171096d92175dda0a560f0955ade9b8a3a07bdcf151f7359611", size = 776545, upload-time = "2026-08-16T22:56:10.159Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/58d4f842895220b793c53fc94a6489705b3665bb4d0ae4d338ce03fdf9fb/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f76d1562643693b8a40066f1f96af795b93fd9bcfc9690a1af2ff4c5867ee29e", size = 1107271, upload-time = "2026-08-16T22:54:50.266Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b8/206468912d2153306bb8a41afdfc59e45b7a73a0495bbe4b9cb4f0e79c1d/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60cab3ab4ea468d31a33739ffd7e94ec3e37dea891d65a6582ecc8a477175191", size = 1156915, upload-time = "2026-08-16T22:54:25.89Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d3/bf5a22929b70a4cfd3edf69c5642b029b27ddb5cfda48fa295d384b01abb/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:22cf19388f0ff6ced8eb3e49c903d14938e4ed909d93bf28383eef451511e424", size = 1281205, upload-time = "2026-08-16T22:54:44.083Z" }, + { url = "https://files.pythonhosted.org/packages/07/a2/d7b2ba444d36fc84d4779f4431e74dd9b023dc63bcf282199f6e48ad39f4/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:057d0232f1224dcd0b7698902551a4341a7399f90670b036db6c4376715fe889", size = 1324243, upload-time = "2026-08-16T22:55:41.123Z" }, + { url = "https://files.pythonhosted.org/packages/d1/95/afe6b531fd01928c6f63d394ee413fa2338d088b2b44efcc23596b54477e/hypothesis-6.165.10-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ab0f2e9d7d7d4db257f7cf53de3706c2baf124269571f20ffc2bcd6781f03063", size = 616382, upload-time = "2026-08-16T22:55:18.449Z" }, + { url = "https://files.pythonhosted.org/packages/48/86/9b4fb75f520a028edec50ffc904a94d724180395d71feb6d7a0ce7bb6f00/hypothesis-6.165.10-cp314-cp314-win_amd64.whl", hash = "sha256:d1ea02fa8ab3d33eb1125eade81f7136341eb429152c6dbe2ae6f8bc33b3fbdd", size = 672145, upload-time = "2026-08-16T22:54:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ba/f7bbaae0c789bab7ddb764d2056ee1a463cc95a8acbccc90d4184e48b242/hypothesis-6.165.10-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ed1a5891e59472884a03cb9875483e8fc131c80a275c60967f8afc5458a0c8ff", size = 783287, upload-time = "2026-08-16T22:54:23.751Z" }, + { url = "https://files.pythonhosted.org/packages/3a/83/01ef80772b4abd335c49405576dc503cede94fb5da30ba2643a119013aea/hypothesis-6.165.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:09772e328a26e50486ac572be34f9887f9aa185efe7ebb16bde4e8f6038db1f4", size = 774991, upload-time = "2026-08-16T22:55:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/a3/0b/f47506241f9d5a5a2efe4c65b6bf4830e9d9576e5d3779007a260699e608/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cf3b612542ba174c9da4000b59a4f4c81e8d66f87509be85d3a1b71b5c36413", size = 1105499, upload-time = "2026-08-16T22:54:51.864Z" }, + { url = "https://files.pythonhosted.org/packages/84/fe/abb3909b7089835112fbe75bf00d817d733b3a8032759783db0a24ff1e56/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69ec5be85ef508e206153bed8eafd03f7995dc464356c8bbb279a1e2b7d56f3", size = 1155685, upload-time = "2026-08-16T22:54:30.94Z" }, + { url = "https://files.pythonhosted.org/packages/73/2f/1964738921640184067121ae77414522fc3f0463fc26c6e25a4f3b8e42ca/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:dd207497bb985918409a1bb5db85d1875f74e1269487332113b73d1ee7c77647", size = 1279177, upload-time = "2026-08-16T22:54:40.179Z" }, + { url = "https://files.pythonhosted.org/packages/34/c5/312af8ae038d3af9cf3f7f1021c1abfe31c0d9035e4cf63519e0a7dc983e/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:00de0abdcf8c05c9d0eab735a3c49a276376b55151e6fcb903c2b39a90e5e5c3", size = 1322921, upload-time = "2026-08-16T22:54:42.7Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e7/b0a2fde7570c090a1b914026266a421c751ef10138fffe37fe0ef9e675c0/hypothesis-6.165.10-cp314-cp314t-win_amd64.whl", hash = "sha256:cc2da5aa4edf14743fa9257e5ba3513963999f01211635702479d8e92b8207c8", size = 672147, upload-time = "2026-08-16T22:55:27.527Z" }, + { url = "https://files.pythonhosted.org/packages/47/fd/985aa564d6ffd06483d45a62b40d319df0a703cd8bc1d041de17d102fbaa/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:eeab73050ea58c13dd56e329f594c1dfe32ebd7bb169bbdf4f8ceefbc31ec6b5", size = 782882, upload-time = "2026-08-16T22:55:37.93Z" }, + { url = "https://files.pythonhosted.org/packages/f8/2c/6cc11151e450f72353a490940cd0db704680d07b78dc75dcc9f480e0d0e1/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:4c68e983d0007d014bb01ad4bcbba78bc432c73a1755ff36d5102ceefa18299a", size = 774584, upload-time = "2026-08-16T22:55:51.822Z" }, + { url = "https://files.pythonhosted.org/packages/10/39/ef26fa79c1738dfe9cdb1a3584fb6717d26429ca6c9d011cc4fdf08130c2/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7730d8197086f65d8969a991d6728a1d420a51b19fea06535c896cb43a1e05d0", size = 1104876, upload-time = "2026-08-16T22:54:58.937Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f4/3fcc84e7637f42bf00d987093b9418083ac8db81b87392608a60f4b7c5fd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7a7980a898a3e6ebe4de1896a0507e3d519edb53fb9b4bda478c9fbeb6514558", size = 1133353, upload-time = "2026-08-16T22:54:28.635Z" }, + { url = "https://files.pythonhosted.org/packages/35/59/21c5c14179c38f8d0de3560e7f1825c083311b3013b63f817d7dc78dfcbd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5820d009aedb7ae9cfd32f98b1ab0c0bbd6268379c4fab042218b6b655c63f8", size = 1132300, upload-time = "2026-08-16T22:56:08.539Z" }, + { url = "https://files.pythonhosted.org/packages/14/af/fbb56059961e416b2de7b9dc5352db2e8572bd5ea46892957e4c1e5548ab/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37a7ac3d34220800e1107871cc391bca1b00439875925d7d821878b8b791f245", size = 1155175, upload-time = "2026-08-16T22:55:19.824Z" }, + { url = "https://files.pythonhosted.org/packages/0f/53/77fb0c2dad445858555429c4e06cf94a59ae8d2407dd6426b5af97c84828/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:dafa7c9dbe3d802f9bcdf261b29c8a70700fb22839947f06e471f62c46b6257f", size = 1109881, upload-time = "2026-08-16T22:55:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7b/d187f673ff30e6ada640953636f978ffe64a6332f756b64163c2277f8d0c/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:90915635b9648071129b0f72c0673cf8eac9eb84cfd445c5bedef30c714b1ec2", size = 1144963, upload-time = "2026-08-16T22:56:13.428Z" }, + { url = "https://files.pythonhosted.org/packages/e0/60/31d504e364134d60af23e5f6365db0da3cf4a51b3ed3d4836e5a2cff12cf/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:e1bbeb7c506b07ee0422cf9b2f7212fefa4240957f03526d38d27bc6743a0a48", size = 1278684, upload-time = "2026-08-16T22:55:22.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e6/89d26834a08c02f8da149e541dd40d7a96f68d9722f43146e69a77436ed7/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:2b36aaffc88625a44f91074c5bbedfdefb9b376c38d1b3c342edcd2e4c8ed16c", size = 1407202, upload-time = "2026-08-16T22:55:14.949Z" }, + { url = "https://files.pythonhosted.org/packages/dc/61/20d1e72246867ea195440092e8bb422c7ddc2f271b87b5b65679d5532719/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:18a3ea838ddea183388f8788750afa8494d79abb5358823be9782585f34445d3", size = 1261395, upload-time = "2026-08-16T22:56:05.448Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9b/ebab6c3c2b90a16abb4119198178652d12aff83cc8ec2cfde5276c69fb1e/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:2a2567b3a03a4a5a7c575c191cfcce321a967df3727803817e75bffbbeaecabe", size = 1279213, upload-time = "2026-08-16T22:55:35.066Z" }, + { url = "https://files.pythonhosted.org/packages/23/78/69b219b524231d36eb20c792e1f01e7cb037e02bd0af1c29f77ed9a969c0/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:8001925fa3dde51cb574e4c9de4c7efe77c4e4d64bd2fd2ef61d5651f9d04f3d", size = 1322367, upload-time = "2026-08-16T22:54:21.279Z" }, + { url = "https://files.pythonhosted.org/packages/55/63/ad5cc153dcc72ae5e7905fb9b3585f3e48ce892a2d6366f90163e867a69d/hypothesis-6.165.10-cp315-abi3.abi3t-win32.whl", hash = "sha256:c6559380469295c4009215fe1cab561301591a3bee2e2fb3f4f96d2273a3affc", size = 666038, upload-time = "2026-08-16T22:56:11.797Z" }, + { url = "https://files.pythonhosted.org/packages/80/32/b62307b73fbc99f0a4381d6f9456df76fbcbb7a27ef7256e26f0376f48ea/hypothesis-6.165.10-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:30797f20ca45e57f526d2df872f63ba453cb4e1091ad542184a7a951af8da79d", size = 671941, upload-time = "2026-08-16T22:55:00.235Z" }, + { url = "https://files.pythonhosted.org/packages/c2/dd/e0f98add0548ef73ea7afac45da1fb8efc854d7f9931db568754d0f963f3/hypothesis-6.165.10-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:c53e9b1c36350df9965ec44d6c0d4e0bbbb38f720dd2b0e1256dc6524d411015", size = 669931, upload-time = "2026-08-16T22:55:50.205Z" }, + { url = "https://files.pythonhosted.org/packages/0b/6a/880d6eeed5c451fb40a66733dadec4a5d498628a4a7f6a8a5f633f4c6dcb/hypothesis-6.165.10-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:34ee6402df6f31274d89119f1561b5f7489c97866afc5b7a3ed3a13d7e762802", size = 784644, upload-time = "2026-08-16T22:54:20.127Z" }, + { url = "https://files.pythonhosted.org/packages/27/e0/9e942bd3c3cf5ea0d5c0fd0905893bbfb6cefb7284c70fcc8033f8fdec38/hypothesis-6.165.10-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:277f41801e88dad2eba082f91a75632b7584ff64044ba2cf9dadf511b0d19cd0", size = 780515, upload-time = "2026-08-16T22:55:04.676Z" }, + { url = "https://files.pythonhosted.org/packages/19/32/f11a618415dc5fa9cdde41fea56c489f0814759527ae1ecd11a75a4558b9/hypothesis-6.165.10-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72df95fb1db41755b155c5f02106e0036a339250555c8d351d488704fd112cf9", size = 1109374, upload-time = "2026-08-16T22:56:00.241Z" }, + { url = "https://files.pythonhosted.org/packages/5e/6f/db49b719842297c2b71e0d81e5b8967d31215fb7389421abcb465ce7ed3f/hypothesis-6.165.10-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e20a02775eb3cf0ffb4f0219b6d7c1f240336663d4e5d7028675ec247c790c4", size = 1159092, upload-time = "2026-08-16T22:55:58.57Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2a/bf0bae84ba1cb3923d295973f1fe38ee867eaf90119e0d559116083be300/hypothesis-6.165.10-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:1ec53f08732e3cfd0342cbbd75dbd1b193c8f19390660466e536a748bb81f757", size = 676045, upload-time = "2026-08-16T22:55:45.514Z" }, +] + [[package]] name = "idna" version = "3.15" @@ -4430,6 +4522,7 @@ dev = [ { name = "diff-cover" }, { name = "fakeredis" }, { name = "fastapi-offline" }, + { name = "hypothesis" }, { name = "keyring" }, { name = "langfuse" }, { name = "openapi-core" }, @@ -4449,6 +4542,7 @@ dev = [ { name = "pytest-rerunfailures" }, { name = "pytest-timeout" }, { name = "pytest-xdist" }, + { name = "reportlab" }, { name = "requests-mock" }, { name = "responses" }, { name = "respx" }, @@ -4615,6 +4709,7 @@ dev = [ { name = "diff-cover", specifier = "==9.7.2" }, { name = "fakeredis", specifier = "==2.34.1" }, { name = "fastapi-offline", specifier = "==1.7.6" }, + { name = "hypothesis", specifier = "==6.165.10" }, { name = "keyring", specifier = "==25.7.0" }, { name = "langfuse", specifier = "==2.59.7" }, { name = "openapi-core", specifier = "==0.22.0" }, @@ -4634,6 +4729,7 @@ dev = [ { name = "pytest-rerunfailures", specifier = "==15.1" }, { name = "pytest-timeout", specifier = "==2.4.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, + { name = "reportlab", specifier = "==5.0.1" }, { name = "requests-mock", specifier = "==1.12.1" }, { name = "responses", specifier = "==0.26.0" }, { name = "respx", specifier = "==0.22.0" }, @@ -8189,6 +8285,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/80/35b4c33c804a165a7f55289afda3ea9e3eb6d15800341a2d66455c0f1f30/regex-2026.5.9-cp314-cp314t-win_arm64.whl", hash = "sha256:5e41809d2683fcde7d5a8c87a6567ba1fb1ce0de9f31bff578de00a4b2d76daa", size = 275713, upload-time = "2026-05-09T23:15:16.98Z" }, ] +[[package]] +name = "reportlab" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/51/dbe28534ae12c852f61be91f039f343305fd1f34f1c66b8de75afae7a525/reportlab-5.0.1.tar.gz", hash = "sha256:ebd13154be1c8515e665de70bd2d303ae9ddc3ef47e44afd5116441ca0283a26", size = 3945711, upload-time = "2026-08-20T13:48:16.461Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/cb/dacbc268cb68d0428ea2cbd85266195a9ab3e677449589ddae59bd7542ac/reportlab-5.0.1-py3-none-any.whl", hash = "sha256:1c36e6bb0e71780c72331eba60da7f602e8d4389a8723825af71342e49d791e8", size = 1957258, upload-time = "2026-08-20T13:48:14.026Z" }, +] + [[package]] name = "requests" version = "2.34.0"