From 26d27803eb487a1b6b7881e2bdb74775b23c824b Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 19 Feb 2026 15:17:27 -0300 Subject: [PATCH 01/54] fix(models): disable function calling for PublicAI Apertus models The Apertus 8B and 70B models do not support standard OpenAI-style tool calling. Per Swiss AI's docs, tool use integration into inference engines is still in development. Set supports_function_calling and supports_tool_choice to false. Fixes #21124 --- model_prices_and_context_window.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 41acb5c8101..0ee2d586f85 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25907,8 +25907,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/swiss-ai/apertus-70b-instruct": { "input_cost_per_token": 0.0, @@ -25919,8 +25919,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/aisingapore/Gemma-SEA-LION-v4-27B-IT": { "input_cost_per_token": 0.0, From 27413790e6155c97ea188e3063f787e7c8bb4a34 Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 19 Feb 2026 16:07:30 -0300 Subject: [PATCH 02/54] fix(openrouter): use provider-reported usage in streaming without stream_options When providers like OpenRouter send a usage chunk after the finish_reason chunk, _hidden_params["usage"] was already calculated (with zeros) before the usage data arrived. The StopIteration handler now recalculates usage from stream_chunk_builder and updates the shared _hidden_params dict so the user's copy reflects the real provider-reported token counts. Fixes #20760 --- .../litellm_core_utils/streaming_handler.py | 34 +++++++ .../test_streaming_handler.py | 89 +++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 7a6752fbff8..df8095e64d8 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -149,6 +149,7 @@ class CustomStreamWrapper: ) # keep track of the returned chunks - used for calculating the input/output tokens for stream options self.is_function_call = self.check_is_function_call(logging_obj=logging_obj) self.created: Optional[int] = None + self._last_returned_hidden_params: Optional[dict] = None def __iter__(self): return self @@ -1787,6 +1788,7 @@ class CustomStreamWrapper: if self.sent_last_chunk is True and self.stream_options is None: usage = calculate_total_usage(chunks=self.chunks) response._hidden_params["usage"] = usage + self._last_returned_hidden_params = response._hidden_params # Add MCP metadata to final chunk if present response = self._add_mcp_metadata_to_final_chunk(response) # RETURN RESULT @@ -1828,6 +1830,24 @@ class CustomStreamWrapper: None, cache_hit, ) + # Update hidden_params with final usage from + # stream_chunk_builder. Some providers (e.g. OpenRouter) + # send usage in a chunk after finish_reason, which arrives + # after _hidden_params["usage"] was initially set. The + # _hidden_params dict is the same object the user received + # (shared by reference), so mutating it here also corrects + # the user's copy. + if ( + self.stream_options is None + and complete_streaming_response is not None + and self._last_returned_hidden_params is not None + ): + final_usage = getattr( + complete_streaming_response, "usage", None + ) + if final_usage is not None: + self._last_returned_hidden_params["usage"] = final_usage + if self.sent_stream_usage is False and self.send_stream_usage is True: self.sent_stream_usage = True return response @@ -1951,6 +1971,7 @@ class CustomStreamWrapper: if self.sent_last_chunk is True and self.stream_options is None: usage = calculate_total_usage(chunks=self.chunks) processed_chunk._hidden_params["usage"] = usage + self._last_returned_hidden_params = processed_chunk._hidden_params # Call post-call streaming deployment hook for final chunk if self.sent_last_chunk is True: @@ -2017,6 +2038,19 @@ class CustomStreamWrapper: cache_hit=cache_hit, ) ) + # Update hidden_params with final usage from + # stream_chunk_builder (see sync __next__ for full comment). + if ( + self.stream_options is None + and complete_streaming_response is not None + and self._last_returned_hidden_params is not None + ): + final_usage = getattr( + complete_streaming_response, "usage", None + ) + if final_usage is not None: + self._last_returned_hidden_params["usage"] = final_usage + if self.sent_stream_usage is False and self.send_stream_usage is True: self.sent_stream_usage = True return response diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index ec2f528a35d..b5922378645 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1185,3 +1185,92 @@ def test_is_chunk_non_empty_with_valid_tool_calls( ) is True ) + + +def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj): + """ + Test that provider-reported usage from a post-finish_reason chunk + is surfaced in _hidden_params even when stream_options is NOT set. + + Reproduces issue #20760: OpenRouter sends a final chunk with usage data + after the finish_reason chunk. The hidden_params["usage"] on the last + user-visible chunk was being calculated before this usage chunk arrived, + resulting in zeros. The fix recalculates it in the StopIteration handler + after stream_chunk_builder processes all chunks. + """ + # Simulate OpenRouter's actual streaming pattern: + # 1) content chunk + # 2) finish_reason chunk (content="") + # 3) usage chunk (content="", finish_reason=None, usage={...}) + chunks = [ + ModelResponseStream( + id="gen-abc", + object="chat.completion.chunk", + created=1000000, + model="openrouter/openai/gpt-4o-mini", + choices=[ + StreamingChoices( + index=0, + delta=Delta(role="assistant", content="Hello"), + finish_reason=None, + ) + ], + ), + ModelResponseStream( + id="gen-abc", + object="chat.completion.chunk", + created=1000000, + model="openrouter/openai/gpt-4o-mini", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content=""), + finish_reason="stop", + ) + ], + ), + ModelResponseStream( + id="gen-abc", + object="chat.completion.chunk", + created=1000000, + model="openrouter/openai/gpt-4o-mini", + choices=[ + StreamingChoices( + index=0, + delta=Delta(role="assistant", content=""), + finish_reason=None, + ) + ], + usage=Usage( + prompt_tokens=20, + completion_tokens=135, + total_tokens=155, + ), + ), + ] + + # Create a CustomStreamWrapper with NO stream_options + wrapper = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=chunks), + model="openrouter/openai/gpt-4o-mini", + logging_obj=logging_obj, + custom_llm_provider="openrouter", + stream_options=None, + ) + + # Consume the stream + collected = [] + for chunk in wrapper: + collected.append(chunk) + + # The last user-visible chunk's _hidden_params["usage"] should + # contain the provider-reported values, not zeros. + last_chunk = collected[-1] + hidden_usage = last_chunk._hidden_params.get("usage") + assert hidden_usage is not None, "Expected usage in _hidden_params" + assert hidden_usage.prompt_tokens == 20, ( + f"Expected prompt_tokens=20 from provider, got {hidden_usage.prompt_tokens}" + ) + assert hidden_usage.completion_tokens == 135, ( + f"Expected completion_tokens=135 from provider, got {hidden_usage.completion_tokens}" + ) From c6f60bed71f9198ed697c698bce533dbb9149106 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Mon, 23 Feb 2026 19:44:30 +0530 Subject: [PATCH 03/54] perf(spendlogs): optimize old spendlog deletion cron job --- .../litellm_proxy_extras/schema.prisma | 1 + .../db_transaction_queue/spend_log_cleanup.py | 30 ++++++------ litellm/proxy/schema.prisma | 1 + .../proxy/test_spend_log_cleanup.py | 47 ++++++------------- 4 files changed, 32 insertions(+), 47 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 4af7484148c..40bcaff67d6 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -500,6 +500,7 @@ model LiteLLM_SpendLogs { agent_id String? proxy_server_request Json? @default("{}") @@index([startTime]) + @@index([startTime, request_id]) @@index([end_user]) @@index([session_id]) } diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index 02fa84bae30..8c59c79ff0a 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -76,27 +76,29 @@ class SpendLogCleanup: "Max logs deleted - 1,00,000, rest of the logs will be deleted in next run" ) break - # Step 1: Find logs to delete - logs_to_delete = await prisma_client.db.litellm_spendlogs.find_many( - where={"startTime": {"lt": cutoff_date}}, - take=self.batch_size, + # Step 1: Find logs and delete them in one go without fetching to application + # Delete in batches, limited by self.batch_size + deleted_count = await prisma_client.db.execute_raw( + """ + DELETE FROM "LiteLLM_SpendLogs" + WHERE "request_id" IN ( + SELECT "request_id" FROM "LiteLLM_SpendLogs" + WHERE "startTime" < $1::timestamptz + LIMIT $2 + ) + """, + cutoff_date, + self.batch_size, ) - verbose_proxy_logger.info(f"Found {len(logs_to_delete)} logs in this batch") + verbose_proxy_logger.info(f"Deleted {deleted_count} logs in this batch") - if not logs_to_delete: + if deleted_count == 0: verbose_proxy_logger.info( f"No more logs to delete. Total deleted: {total_deleted}" ) break - request_ids = [log.request_id for log in logs_to_delete] - - # Step 2: Delete them in one go - await prisma_client.db.litellm_spendlogs.delete_many( - where={"request_id": {"in": request_ids}} - ) - - total_deleted += len(logs_to_delete) + total_deleted += deleted_count run_count += 1 # Add a small sleep to prevent overwhelming the database diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 50c0a55a875..7a1010d95c0 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -499,6 +499,7 @@ model LiteLLM_SpendLogs { agent_id String? proxy_server_request Json? @default("{}") @@index([startTime]) + @@index([startTime, request_id]) @@index([end_user]) @@index([session_id]) } diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py index 1ffbb83caef..c1fa3ad0c43 100644 --- a/tests/test_litellm/proxy/test_spend_log_cleanup.py +++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py @@ -151,28 +151,16 @@ async def test_should_delete_spend_logs(): @pytest.mark.asyncio async def test_cleanup_old_spend_logs_batch_deletion(): - from types import SimpleNamespace - from unittest.mock import AsyncMock, MagicMock, patch + from unittest.mock import AsyncMock, MagicMock # Setup Prisma client mock_prisma_client = MagicMock() mock_db = MagicMock() - # Mock spendlogs table - mock_spendlogs = MagicMock() - mock_spendlogs.find_many = AsyncMock() - mock_spendlogs.delete_many = AsyncMock() - - # Create 1500 mocked logs with .request_id - mock_logs = [SimpleNamespace(request_id=f"req_{i}") for i in range(1500)] - mock_spendlogs.find_many.side_effect = [ - mock_logs[:1000], # Batch 1 - mock_logs[1000:], # Batch 2 - [], # Done - ] + # Mock execute_raw to return deleted counts + mock_db.execute_raw = AsyncMock(side_effect=[1000, 500, 0]) # Wire up mocks - mock_db.litellm_spendlogs = mock_spendlogs mock_prisma_client.db = mock_db # Mock Redis cache and pod_lock_manager @@ -189,15 +177,13 @@ async def test_cleanup_old_spend_logs_batch_deletion(): assert cleaner._should_delete_spend_logs() is True await cleaner.cleanup_old_spend_logs(mock_prisma_client) - # Validate batching and deletion - assert mock_spendlogs.find_many.call_count == 3 - assert mock_spendlogs.delete_many.call_count == 2 - mock_spendlogs.delete_many.assert_any_call( - where={"request_id": {"in": [f"req_{i}" for i in range(1000)]}} - ) - mock_spendlogs.delete_many.assert_any_call( - where={"request_id": {"in": [f"req_{i}" for i in range(1000, 1500)]}} - ) + # Validate batching and deletion via raw SQL + assert mock_db.execute_raw.call_count == 3 + + # Check the first call argument + call_args_sql = mock_db.execute_raw.call_args_list[0][0][0] + assert 'DELETE FROM "LiteLLM_SpendLogs"' in call_args_sql + assert 'WHERE "request_id" IN' in call_args_sql @pytest.mark.asyncio @@ -208,10 +194,7 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff(): # Setup Prisma client mock_prisma_client = MagicMock() mock_db = MagicMock() - mock_spendlogs = MagicMock() - mock_spendlogs.find_many = AsyncMock(return_value=[]) - mock_spendlogs.delete_many = AsyncMock() - mock_db.litellm_spendlogs = mock_spendlogs + mock_db.execute_raw = AsyncMock(return_value=0) mock_prisma_client.db = mock_db # Mock Redis cache and pod_lock_manager @@ -229,7 +212,7 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff(): await cleaner.cleanup_old_spend_logs(mock_prisma_client) # Verify the cutoff date is correct - cutoff_date = mock_spendlogs.find_many.call_args[1]["where"]["startTime"]["lt"] + cutoff_date = mock_db.execute_raw.call_args[0][1] expected_cutoff = datetime.now(timezone.utc) - timedelta(seconds=86400) assert ( abs((cutoff_date - expected_cutoff).total_seconds()) < 1 @@ -242,14 +225,12 @@ async def test_cleanup_old_spend_logs_no_retention_period(): Test that no logs are deleted when no retention period is set """ mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_spendlogs.find_many = AsyncMock() - mock_prisma_client.db.litellm_spendlogs.delete = AsyncMock() + mock_prisma_client.db.execute_raw = AsyncMock() cleaner = SpendLogCleanup(general_settings={}) # no retention await cleaner.cleanup_old_spend_logs(mock_prisma_client) - mock_prisma_client.db.litellm_spendlogs.find_many.assert_not_called() - mock_prisma_client.db.litellm_spendlogs.delete.assert_not_called() + mock_prisma_client.db.execute_raw.assert_not_called() def test_cleanup_batch_size_env_var(monkeypatch): From f453427264866e038d9b714701fbce925e0fe6c6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 25 Feb 2026 17:19:33 +0530 Subject: [PATCH 04/54] Add v1 for anthropic responses transformation --- .../messages/handler.py | 59 ++- .../responses_adapters/__init__.py | 3 + .../responses_adapters/handler.py | 213 +++++++++ .../responses_adapters/streaming_iterator.py | 265 ++++++++++++ .../responses_adapters/transformation.py | 407 ++++++++++++++++++ ...erimental_pass_through_messages_handler.py | 62 +-- 6 files changed, 958 insertions(+), 51 deletions(-) create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 7e5a4f22a7f..6fe0fcd4fdf 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -25,8 +25,18 @@ from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler +from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler from .utils import AnthropicMessagesRequestUtils, mock_response +# Providers that are routed directly to the OpenAI Responses API instead of +# going through chat/completions. +_RESPONSES_API_PROVIDERS = frozenset({"openai", "azure", "azure_text"}) + + +def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool: + """Return True when the provider should use the Responses API path.""" + return custom_llm_provider in _RESPONSES_API_PROVIDERS + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -282,29 +292,34 @@ def anthropic_messages_handler( ) ) if anthropic_messages_provider_config is None: - # Handle non-Anthropic models using the adapter - return ( - LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( - max_tokens=max_tokens, - messages=messages, - model=model, - metadata=metadata, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - _is_async=is_async, - api_key=api_key, - api_base=api_base, - client=client, - custom_llm_provider=custom_llm_provider, - **kwargs, + # Route to Responses API for OpenAI / Azure, chat/completions for everything else. + _shared_kwargs = dict( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + _is_async=is_async, + api_key=api_key, + api_base=api_base, + client=client, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + if _should_route_to_responses_api(custom_llm_provider): + return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + **_shared_kwargs ) + return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( + **_shared_kwargs ) if custom_llm_provider is None: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py new file mode 100644 index 00000000000..6ad3c7b0164 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py @@ -0,0 +1,3 @@ +from .transformation import LiteLLMAnthropicToResponsesAPIAdapter + +__all__ = ["LiteLLMAnthropicToResponsesAPIAdapter"] diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py new file mode 100644 index 00000000000..18dbabb1e14 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -0,0 +1,213 @@ +""" +Handler for the Anthropic v1/messages -> OpenAI Responses API path. + +Used when the target model is an OpenAI or Azure model. +""" + +from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union + +import litellm +from litellm.types.llms.anthropic import AnthropicMessagesRequest +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, +) +from litellm.types.llms.openai import ResponsesAPIResponse + +from .streaming_iterator import AnthropicResponsesStreamWrapper +from .transformation import LiteLLMAnthropicToResponsesAPIAdapter + +_ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter() + + +def _build_responses_kwargs( + *, + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + extra_kwargs: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). + """ + # Build a typed AnthropicMessagesRequest for the adapter + request_data: Dict[str, Any] = {"model": model, "messages": messages, "max_tokens": max_tokens} + if metadata: + request_data["metadata"] = metadata + if system: + request_data["system"] = system + if temperature is not None: + request_data["temperature"] = temperature + if thinking: + request_data["thinking"] = thinking + if tool_choice: + request_data["tool_choice"] = tool_choice + if tools: + request_data["tools"] = tools + if top_p is not None: + request_data["top_p"] = top_p + if output_format: + request_data["output_format"] = output_format + + anthropic_request = AnthropicMessagesRequest(**request_data) + responses_kwargs = _ADAPTER.translate_request(anthropic_request) + + if stream: + responses_kwargs["stream"] = True + + # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) + excluded = {"anthropic_messages"} + for key, value in (extra_kwargs or {}).items(): + if key == "litellm_logging_obj" and value is not None: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObject, + ) + from litellm.types.utils import CallTypes + + if isinstance(value, LiteLLMLoggingObject): + # Reclassify as acompletion so the success handler doesn't try to + # validate the Responses API event as an AnthropicResponse. + # (Mirrors the pattern used in LiteLLMMessagesToCompletionTransformationHandler.) + setattr(value, "call_type", CallTypes.acompletion.value) + responses_kwargs[key] = value + elif key not in excluded and key not in responses_kwargs and value is not None: + responses_kwargs[key] = value + + return responses_kwargs + + +class LiteLLMMessagesToResponsesAPIHandler: + """ + Handles Anthropic /v1/messages requests for OpenAI / Azure models by + calling litellm.responses() / litellm.aresponses() directly and translating + the response back to Anthropic format. + """ + + @staticmethod + async def async_anthropic_messages_handler( + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + **kwargs, + ) -> Union[AnthropicMessagesResponse, AsyncIterator]: + responses_kwargs = _build_responses_kwargs( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + result = await litellm.aresponses(**responses_kwargs) + + if stream: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + return wrapper.async_anthropic_sse_wrapper() + + if not isinstance(result, ResponsesAPIResponse): + raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") + + return _ADAPTER.translate_response(result) + + @staticmethod + def anthropic_messages_handler( + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + _is_async: bool = False, + **kwargs, + ) -> Union[ + AnthropicMessagesResponse, + AsyncIterator[Any], + Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]], + ]: + if _is_async: + return LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + **kwargs, + ) + + # Sync path + responses_kwargs = _build_responses_kwargs( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + result = litellm.responses(**responses_kwargs) + + if stream: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + return wrapper.async_anthropic_sse_wrapper() + + if not isinstance(result, ResponsesAPIResponse): + raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") + + return _ADAPTER.translate_response(result) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py new file mode 100644 index 00000000000..0e6268e82f3 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -0,0 +1,265 @@ +# What is this? +## Translates OpenAI call to Anthropic `/v1/messages` format +import json +import traceback +from collections import deque +from typing import Any, AsyncIterator, Dict + +from litellm import verbose_logger +from litellm._uuid import uuid + + +class AnthropicResponsesStreamWrapper: + """ + Wraps a Responses API streaming iterator and re-emits events in Anthropic SSE format. + + Responses API event flow (relevant subset): + response.created -> message_start + response.output_item.added -> content_block_start (if message/function_call) + response.output_text.delta -> content_block_delta (text_delta) + response.reasoning_summary_text.delta -> content_block_delta (thinking_delta) + response.function_call_arguments.delta -> content_block_delta (input_json_delta) + response.output_item.done -> content_block_stop + response.completed -> message_delta + message_stop + """ + + def __init__( + self, + responses_stream: Any, + model: str, + ) -> None: + self.responses_stream = responses_stream + self.model = model + self._message_id: str = f"msg_{uuid.uuid4()}" + self._current_block_index: int = -1 + # Map item_id -> content_block_index so we can stop the right block later + self._item_id_to_block_index: Dict[str, int] = {} + # Track open function_call items by item_id so we can emit tool_use start + self._pending_tool_ids: Dict[str, str] = {} # item_id -> call_id / name accumulator + self._sent_message_start = False + self._sent_message_stop = False + self._chunk_queue: deque = deque() + + def _make_message_start(self) -> Dict[str, Any]: + return { + "type": "message_start", + "message": { + "id": self._message_id, + "type": "message", + "role": "assistant", + "content": [], + "model": self.model, + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, + } + + def _next_block_index(self) -> int: + self._current_block_index += 1 + return self._current_block_index + + def _process_event(self, event: Any) -> None: + """Convert one Responses API event into zero or more Anthropic chunks queued for emission.""" + event_type = getattr(event, "type", None) + if event_type is None and isinstance(event, dict): + event_type = event.get("type") + + if event_type is None: + return + + # ---- message_start ---- + if event_type == "response.created": + self._sent_message_start = True + self._chunk_queue.append(self._make_message_start()) + return + + # ---- content_block_start for a new output message item ---- + if event_type == "response.output_item.added": + item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None) + if item is None: + return + item_type = getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None) + item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) + + if item_type == "message": + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": {"type": "text", "text": ""}, + }) + elif item_type == "function_call": + call_id = getattr(item, "call_id", None) or (item.get("call_id") if isinstance(item, dict) else None) or "" + name = getattr(item, "name", None) or (item.get("name") if isinstance(item, dict) else None) or "" + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._pending_tool_ids[item_id] = call_id + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": { + "type": "tool_use", + "id": call_id, + "name": name, + "input": {}, + }, + }) + elif item_type == "reasoning": + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": {"type": "thinking", "thinking": ""}, + }) + return + + # ---- text delta ---- + if event_type == "response.output_text.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "text_delta", "text": delta}, + }) + return + + # ---- reasoning summary text delta ---- + if event_type == "response.reasoning_summary_text.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "thinking_delta", "thinking": delta}, + }) + return + + # ---- function call arguments delta ---- + if event_type == "response.function_call_arguments.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "input_json_delta", "partial_json": delta}, + }) + return + + # ---- output item done -> content_block_stop ---- + if event_type == "response.output_item.done": + item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None) + item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_stop", + "index": block_idx, + }) + return + + # ---- response completed -> message_delta + message_stop ---- + if event_type in ("response.completed", "response.failed", "response.incomplete"): + response_obj = getattr(event, "response", None) or (event.get("response") if isinstance(event, dict) else None) + stop_reason = "end_turn" + input_tokens = 0 + output_tokens = 0 + cache_creation_tokens = 0 + cache_read_tokens = 0 + + if response_obj is not None: + status = getattr(response_obj, "status", None) + if status == "incomplete": + stop_reason = "max_tokens" + usage = getattr(response_obj, "usage", None) + if usage is not None: + input_tokens = getattr(usage, "input_tokens", 0) or 0 + output_tokens = getattr(usage, "output_tokens", 0) or 0 + cache_creation_tokens = getattr(usage, "input_tokens_details", None) + cache_read_tokens = getattr(usage, "output_tokens_details", None) + # Prefer direct cache fields if present + cache_creation_tokens = getattr(usage, "cache_creation_input_tokens", 0) or 0 + cache_read_tokens = getattr(usage, "cache_read_input_tokens", 0) or 0 + + # Check if tool_use was in the output to override stop_reason + if response_obj is not None: + output = getattr(response_obj, "output", []) or [] + for out_item in output: + out_type = getattr(out_item, "type", None) or (out_item.get("type") if isinstance(out_item, dict) else None) + if out_type == "function_call": + stop_reason = "tool_use" + break + + usage_delta: Dict[str, Any] = { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } + if cache_creation_tokens: + usage_delta["cache_creation_input_tokens"] = cache_creation_tokens + if cache_read_tokens: + usage_delta["cache_read_input_tokens"] = cache_read_tokens + + self._chunk_queue.append({ + "type": "message_delta", + "delta": {"stop_reason": stop_reason, "stop_sequence": None}, + "usage": usage_delta, + }) + self._chunk_queue.append({"type": "message_stop"}) + self._sent_message_stop = True + return + + def __aiter__(self) -> "AnthropicResponsesStreamWrapper": + return self + + async def __anext__(self) -> Dict[str, Any]: + # Return any queued chunks first + if self._chunk_queue: + return self._chunk_queue.popleft() + + # Emit message_start if not yet done (fallback if response.created wasn't fired) + if not self._sent_message_start: + self._sent_message_start = True + self._chunk_queue.append(self._make_message_start()) + return self._chunk_queue.popleft() + + # Consume the upstream stream + try: + async for event in self.responses_stream: + self._process_event(event) + if self._chunk_queue: + return self._chunk_queue.popleft() + except StopAsyncIteration: + pass + except Exception as e: + verbose_logger.error( + f"AnthropicResponsesStreamWrapper error: {e}\n{traceback.format_exc()}" + ) + + # Drain any remaining queued chunks + if self._chunk_queue: + return self._chunk_queue.popleft() + + raise StopAsyncIteration + + async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]: + """Yield SSE-encoded bytes for each Anthropic event chunk.""" + async for chunk in self: + if isinstance(chunk, dict): + event_type: str = str(chunk.get("type", "message")) + payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n" + yield payload.encode() + else: + yield chunk diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py new file mode 100644 index 00000000000..a428e8f4e8f --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -0,0 +1,407 @@ +""" +Transformation layer: Anthropic /v1/messages <-> OpenAI Responses API. + +This module owns all format conversions for the direct v1/messages -> Responses API +path used for OpenAI and Azure models. +""" + +import json +from typing import Any, Dict, List, Optional, Union, cast + +from litellm.types.llms.anthropic import ( + AllAnthropicToolsValues, + AnthopicMessagesAssistantMessageParam, + AnthropicFinishReason, + AnthropicMessagesRequest, + AnthropicMessagesToolChoice, + AnthropicMessagesUserMessageParam, + AnthropicResponseContentBlockText, + AnthropicResponseContentBlockThinking, + AnthropicResponseContentBlockToolUse, +) +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, + AnthropicUsage, +) +from litellm.types.llms.openai import ResponsesAPIResponse + + +class LiteLLMAnthropicToResponsesAPIAdapter: + """ + Converts Anthropic /v1/messages requests to OpenAI Responses API format and + converts Responses API responses back to Anthropic format. + """ + + # ------------------------------------------------------------------ # + # Request translation: Anthropic -> Responses API # + # ------------------------------------------------------------------ # + + @staticmethod + def _translate_anthropic_image_source_to_url(source: dict) -> Optional[str]: + """Convert Anthropic image source to a URL string.""" + source_type = source.get("type") + if source_type == "base64": + media_type = source.get("media_type", "image/jpeg") + data = source.get("data", "") + return f"data:{media_type};base64,{data}" if data else None + elif source_type == "url": + return source.get("url") + return None + + def translate_messages_to_responses_input( + self, + messages: List[ + Union[ + AnthropicMessagesUserMessageParam, + AnthopicMessagesAssistantMessageParam, + ] + ], + ) -> List[Dict[str, Any]]: + """ + Convert Anthropic messages list to Responses API `input` items. + + Mapping: + user text -> message(role=user, input_text) + user image -> message(role=user, input_image) + user tool_result -> function_call_output + assistant text -> message(role=assistant, output_text) + assistant tool_use -> function_call + """ + input_items: List[Dict[str, Any]] = [] + + for m in messages: + role = m["role"] + content = m.get("content") + + if role == "user": + if isinstance(content, str): + input_items.append({ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": content}], + }) + elif isinstance(content, list): + user_parts: List[Dict[str, Any]] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "text": + user_parts.append({"type": "input_text", "text": block.get("text", "")}) + elif btype == "image": + url = self._translate_anthropic_image_source_to_url(block.get("source", {})) + if url: + user_parts.append({"type": "input_image", "image_url": url}) + elif btype == "tool_result": + tool_use_id = block.get("tool_use_id", "") + inner = block.get("content") + if inner is None: + output_text = "" + elif isinstance(inner, str): + output_text = inner + elif isinstance(inner, list): + parts = [ + c.get("text", "") + for c in inner + if isinstance(c, dict) and c.get("type") == "text" + ] + output_text = "\n".join(parts) + else: + output_text = str(inner) + # tool_result is a top-level item, not inside the message + input_items.append({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": output_text, + }) + if user_parts: + input_items.append({ + "type": "message", + "role": "user", + "content": user_parts, + }) + + elif role == "assistant": + if isinstance(content, str): + input_items.append({ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": content}], + }) + elif isinstance(content, list): + asst_parts: List[Dict[str, Any]] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "text": + asst_parts.append({"type": "output_text", "text": block.get("text", "")}) + elif btype == "tool_use": + # tool_use becomes a top-level function_call item + input_items.append({ + "type": "function_call", + "call_id": block.get("id", ""), + "name": block.get("name", ""), + "arguments": json.dumps(block.get("input", {})), + }) + elif btype == "thinking": + thinking_text = block.get("thinking", "") + if thinking_text: + asst_parts.append({"type": "output_text", "text": thinking_text}) + if asst_parts: + input_items.append({ + "type": "message", + "role": "assistant", + "content": asst_parts, + }) + + return input_items + + def translate_tools_to_responses_api( + self, + tools: List[AllAnthropicToolsValues], + ) -> List[Dict[str, Any]]: + """Convert Anthropic tool definitions to Responses API function tools.""" + result: List[Dict[str, Any]] = [] + for tool in tools: + tool_dict = cast(Dict[str, Any], tool) + tool_type = tool_dict.get("type", "") + tool_name = tool_dict.get("name", "") + # web_search tool + if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search": + result.append({"type": "web_search_preview"}) + continue + func_tool: Dict[str, Any] = {"type": "function", "name": tool_name} + if "description" in tool_dict: + func_tool["description"] = tool_dict["description"] + if "input_schema" in tool_dict: + func_tool["parameters"] = tool_dict["input_schema"] + result.append(func_tool) + return result + + @staticmethod + def translate_tool_choice_to_responses_api( + tool_choice: AnthropicMessagesToolChoice, + ) -> Dict[str, Any]: + """Convert Anthropic tool_choice to Responses API tool_choice.""" + tc_type = tool_choice.get("type") + if tc_type == "any": + return {"type": "required"} + elif tc_type == "tool": + return {"type": "function", "name": tool_choice.get("name", "")} + return {"type": "auto"} + + @staticmethod + def translate_thinking_to_reasoning(thinking: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """ + Convert Anthropic thinking param to Responses API reasoning param. + + thinking.budget_tokens maps to reasoning effort: + >= 10000 -> high, >= 5000 -> medium, >= 2000 -> low, < 2000 -> minimal + """ + if not isinstance(thinking, dict) or thinking.get("type") != "enabled": + return None + budget = thinking.get("budget_tokens", 0) + if budget >= 10000: + effort = "high" + elif budget >= 5000: + effort = "medium" + elif budget >= 2000: + effort = "low" + else: + effort = "minimal" + return {"effort": effort, "summary": "detailed"} + + def translate_request( + self, + anthropic_request: AnthropicMessagesRequest, + ) -> Dict[str, Any]: + """ + Translate a full Anthropic /v1/messages request dict to + litellm.responses() / litellm.aresponses() kwargs. + """ + model: str = anthropic_request["model"] + messages_list = cast( + List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]], + anthropic_request["messages"], + ) + + responses_kwargs: Dict[str, Any] = { + "model": model, + "input": self.translate_messages_to_responses_input(messages_list), + } + + # system -> instructions + system = anthropic_request.get("system") + if system: + if isinstance(system, str): + responses_kwargs["instructions"] = system + elif isinstance(system, list): + text_parts = [ + b.get("text", "") + for b in system + if isinstance(b, dict) and b.get("type") == "text" + ] + responses_kwargs["instructions"] = "\n".join(filter(None, text_parts)) + + # max_tokens -> max_output_tokens + max_tokens = anthropic_request.get("max_tokens") + if max_tokens: + responses_kwargs["max_output_tokens"] = max_tokens + + # temperature / top_p passed through + if "temperature" in anthropic_request: + responses_kwargs["temperature"] = anthropic_request["temperature"] + if "top_p" in anthropic_request: + responses_kwargs["top_p"] = anthropic_request["top_p"] + + # tools + tools = anthropic_request.get("tools") + if tools: + responses_kwargs["tools"] = self.translate_tools_to_responses_api( + cast(List[AllAnthropicToolsValues], tools) + ) + + # tool_choice + tool_choice = anthropic_request.get("tool_choice") + if tool_choice: + responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api( + cast(AnthropicMessagesToolChoice, tool_choice) + ) + + # thinking -> reasoning + thinking = anthropic_request.get("thinking") + if isinstance(thinking, dict): + reasoning = self.translate_thinking_to_reasoning(thinking) + if reasoning: + responses_kwargs["reasoning"] = reasoning + + # output_format -> text format + output_format = anthropic_request.get("output_format") + if isinstance(output_format, dict) and output_format.get("type") == "json_schema": + schema = output_format.get("schema") + if schema: + responses_kwargs["text"] = { + "format": { + "type": "json_schema", + "name": "structured_output", + "schema": schema, + "strict": True, + } + } + + # metadata user_id -> user + metadata = anthropic_request.get("metadata") + if isinstance(metadata, dict) and "user_id" in metadata: + responses_kwargs["user"] = metadata["user_id"] + + return responses_kwargs + + # ------------------------------------------------------------------ # + # Response translation: Responses API -> Anthropic # + # ------------------------------------------------------------------ # + + def translate_response( + self, + response: ResponsesAPIResponse, + ) -> AnthropicMessagesResponse: + """ + Translate an OpenAI ResponsesAPIResponse to AnthropicMessagesResponse. + """ + from openai.types.responses import ( + ResponseFunctionToolCall, + ResponseOutputMessage, + ResponseReasoningItem, + ) + + from litellm.types.llms.openai import ResponseAPIUsage + + content: List[Dict[str, Any]] = [] + stop_reason: AnthropicFinishReason = "end_turn" + + for item in response.output: + if isinstance(item, ResponseReasoningItem): + for summary in item.summary: + text = getattr(summary, "text", "") + if text: + content.append( + AnthropicResponseContentBlockThinking( + type="thinking", + thinking=text, + signature=None, + ).model_dump() + ) + + elif isinstance(item, ResponseOutputMessage): + for part in item.content: + if getattr(part, "type", None) == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=getattr(part, "text", "") + ).model_dump() + ) + + elif isinstance(item, ResponseFunctionToolCall): + try: + input_data = json.loads(item.arguments) if item.arguments else {} + except (json.JSONDecodeError, TypeError): + input_data = {} + content.append( + AnthropicResponseContentBlockToolUse( + type="tool_use", + id=item.call_id or item.id, + name=item.name, + input=input_data, + ).model_dump() + ) + stop_reason = "tool_use" + + elif isinstance(item, dict): + item_type = item.get("type") + if item_type == "message": + for part in item.get("content", []): + if isinstance(part, dict) and part.get("type") == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=part.get("text", "") + ).model_dump() + ) + elif item_type == "function_call": + try: + input_data = json.loads(item.get("arguments", "{}")) + except (json.JSONDecodeError, TypeError): + input_data = {} + content.append( + AnthropicResponseContentBlockToolUse( + type="tool_use", + id=item.get("call_id") or item.get("id", ""), + name=item.get("name", ""), + input=input_data, + ).model_dump() + ) + stop_reason = "tool_use" + + # status -> stop_reason override + if response.status == "incomplete": + stop_reason = "max_tokens" + + # usage + raw_usage: Optional[ResponseAPIUsage] = response.usage + input_tokens = int(getattr(raw_usage, "input_tokens", 0) or 0) + output_tokens = int(getattr(raw_usage, "output_tokens", 0) or 0) + + anthropic_usage = AnthropicUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + return AnthropicMessagesResponse( + id=response.id, + type="message", + role="assistant", + model=response.model or "unknown-model", + stop_sequence=None, + usage=anthropic_usage, # type: ignore + content=content, # type: ignore + stop_reason=stop_reason, + ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 77c74a7847e..c671d9b37b8 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -16,13 +16,14 @@ from litellm.types.utils import Delta, ModelResponse, StreamingChoices def test_anthropic_experimental_pass_through_messages_handler(): """ - Test that api key is passed to litellm.completion + Test that api key is passed to litellm.responses for OpenAI models. + OpenAI and Azure models are routed directly to the Responses API. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=100, @@ -32,19 +33,20 @@ def test_anthropic_experimental_pass_through_messages_handler(): ) except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" + mock_responses.assert_called_once() + assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_and_api_base_and_custom_values(): """ - Test that api key is passed to litellm.completion + Test that api key, api base, and extra kwargs are forwarded to litellm.responses for Azure models. + Azure models are routed directly to the Responses API. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=100, @@ -56,10 +58,10 @@ def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_an ) except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" - assert mock_completion.call_args.kwargs["api_base"] == "test-api-base" - assert mock_completion.call_args.kwargs["custom_key"] == "custom_value" + mock_responses.assert_called_once() + assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" + assert mock_responses.call_args.kwargs["api_base"] == "test-api-base" + assert mock_responses.call_args.kwargs["custom_key"] == "custom_value" def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): @@ -143,19 +145,19 @@ async def test_bedrock_converse_budget_tokens_preserved(): assert thinking_param.get("budget_tokens") == 1024, f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" -def test_openai_model_with_thinking_converts_to_reasoning_effort(): +def test_openai_model_with_thinking_converts_to_reasoning(): """ - Test that when using a non-Anthropic model (like OpenAI gpt-5.2) with thinking parameter, - the thinking is converted to reasoning_effort and NOT passed as thinking. - - This ensures we don't regress on issue #16052 where non-Anthropic models would fail - with UnsupportedParamsError when thinking was passed directly. + Test that when using an OpenAI model with thinking parameter, the thinking is + converted to a Responses API `reasoning` param (NOT passed as thinking). + + OpenAI models are routed directly to the Responses API, so we verify that + litellm.responses() is called with `reasoning` properly set. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=1024, @@ -170,20 +172,22 @@ def test_openai_model_with_thinking_converts_to_reasoning_effort(): except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - - call_kwargs = mock_completion.call_args.kwargs - - # Verify reasoning_effort is set (converted from thinking) - assert "reasoning_effort" in call_kwargs, "reasoning_effort should be passed to completion" + mock_responses.assert_called_once() - # reasoning_effort is transformed into a dict with effort and summary fields - expected_reasoning_effort = {"effort": "minimal", "summary": "detailed"} - assert call_kwargs["reasoning_effort"] == expected_reasoning_effort, \ - f"reasoning_effort should be {expected_reasoning_effort} for budget_tokens=1024, got {call_kwargs.get('reasoning_effort')}" + call_kwargs = mock_responses.call_args.kwargs - # Verify thinking is NOT passed (non-Claude model) - assert "thinking" not in call_kwargs, "thinking should NOT be passed for non-Claude models" + # Verify reasoning is set (converted from thinking) + assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses" + + # budget_tokens=1024 -> effort="minimal" (< 2000 threshold) + expected_reasoning = {"effort": "minimal", "summary": "detailed"} + assert call_kwargs["reasoning"] == expected_reasoning, ( + f"reasoning should be {expected_reasoning} for budget_tokens=1024, " + f"got {call_kwargs.get('reasoning')}" + ) + + # Verify thinking is NOT passed directly to the Responses API + assert "thinking" not in call_kwargs, "thinking should NOT be passed directly to litellm.responses" class TestThinkingParameterTransformation: From d751fdc900887979352a915c8e931c96fef0029d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 11:13:26 +0530 Subject: [PATCH 05/54] add ChatCompletionImageObject in OpenAIChatCompletionAssistantMessage --- litellm/types/llms/openai.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 15e8d1be930..24b18cc488a 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -699,7 +699,15 @@ class OpenAIChatCompletionAssistantMessage(TypedDict, total=False): role: Required[Literal["assistant"]] content: Optional[ Union[ - str, Iterable[Union[ChatCompletionTextObject, ChatCompletionThinkingBlock]] + str, + Iterable[ + Union[ + ChatCompletionTextObject, + ChatCompletionThinkingBlock, + ChatCompletionRedactedThinkingBlock, + ChatCompletionImageObject, + ] + ], ] ] name: Optional[str] From f1080a7e3048ab580acb362cbdb6cb1da460f565 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 11:29:34 +0530 Subject: [PATCH 06/54] Add 'image_url; to both if the intent is to support it in assistant messages --- litellm/types/llms/openai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 24b18cc488a..c0aae9bc2de 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -794,17 +794,19 @@ ValidUserMessageContentTypes = [ "file", ] # used for validating user messages. Prevent users from accidentally sending anthropic messages. -# Assistant message content types (text, thinking, redacted_thinking) +# Assistant message content types (text, thinking, redacted_thinking, image_url) ValidAssistantMessageContentTypesLiteral = Literal[ "text", "thinking", "redacted_thinking", + "image_url", ] ValidAssistantMessageContentTypes = [ "text", "thinking", "redacted_thinking", + "image_url", ] # Combined valid content types for chat completion messages From 7adaf49db7a0fb413704666fca92aaa6b2ad2887 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 18:05:20 +0530 Subject: [PATCH 07/54] Add tranlation of context_management --- .../responses_adapters/handler.py | 16 +++++++ .../responses_adapters/transformation.py | 47 ++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py index 18dbabb1e14..c268d6c5be8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -24,7 +24,9 @@ def _build_responses_kwargs( max_tokens: int, messages: List[Dict], model: str, + context_management: Optional[Dict] = None, metadata: Optional[Dict] = None, + output_config: Optional[Dict] = None, stop_sequences: Optional[List[str]] = None, stream: Optional[bool] = False, system: Optional[str] = None, @@ -42,6 +44,10 @@ def _build_responses_kwargs( """ # Build a typed AnthropicMessagesRequest for the adapter request_data: Dict[str, Any] = {"model": model, "messages": messages, "max_tokens": max_tokens} + if context_management: + request_data["context_management"] = context_management + if output_config: + request_data["output_config"] = output_config if metadata: request_data["metadata"] = metadata if system: @@ -98,7 +104,9 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: List[Dict], model: str, + context_management: Optional[Dict] = None, metadata: Optional[Dict] = None, + output_config: Optional[Dict] = None, stop_sequences: Optional[List[str]] = None, stream: Optional[bool] = False, system: Optional[str] = None, @@ -115,7 +123,9 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens=max_tokens, messages=messages, model=model, + context_management=context_management, metadata=metadata, + output_config=output_config, stop_sequences=stop_sequences, stream=stream, system=system, @@ -145,7 +155,9 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: List[Dict], model: str, + context_management: Optional[Dict] = None, metadata: Optional[Dict] = None, + output_config: Optional[Dict] = None, stop_sequences: Optional[List[str]] = None, stream: Optional[bool] = False, system: Optional[str] = None, @@ -168,7 +180,9 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens=max_tokens, messages=messages, model=model, + context_management=context_management, metadata=metadata, + output_config=output_config, stop_sequences=stop_sequences, stream=stream, system=system, @@ -187,7 +201,9 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens=max_tokens, messages=messages, model=model, + context_management=context_management, metadata=metadata, + output_config=output_config, stop_sequences=stop_sequences, stream=stream, system=system, diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index a428e8f4e8f..c2752272905 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -191,6 +191,37 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return {"type": "function", "name": tool_choice.get("name", "")} return {"type": "auto"} + @staticmethod + def translate_context_management_to_responses_api( + context_management: Dict[str, Any], + ) -> Optional[List[Dict[str, Any]]]: + """ + Convert Anthropic context_management dict to OpenAI Responses API array format. + + Anthropic format: {"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]} + OpenAI format: [{"type": "compaction", "compact_threshold": 150000}] + """ + if not isinstance(context_management, dict): + return None + + edits = context_management.get("edits", []) + if not isinstance(edits, list): + return None + + result: List[Dict[str, Any]] = [] + for edit in edits: + if not isinstance(edit, dict): + continue + edit_type = edit.get("type", "") + if edit_type == "compact_20260112": + entry: Dict[str, Any] = {"type": "compaction"} + trigger = edit.get("trigger") + if isinstance(trigger, dict) and trigger.get("value") is not None: + entry["compact_threshold"] = int(trigger["value"]) + result.append(entry) + + return result if result else None + @staticmethod def translate_thinking_to_reasoning(thinking: Dict[str, Any]) -> Optional[Dict[str, Any]]: """ @@ -276,8 +307,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if reasoning: responses_kwargs["reasoning"] = reasoning - # output_format -> text format + # output_format / output_config.format -> text format + # output_format: {"type": "json_schema", "schema": {...}} + # output_config: {"format": {"type": "json_schema", "schema": {...}}} output_format = anthropic_request.get("output_format") + output_config = anthropic_request.get("output_config") + if not isinstance(output_format, dict) and isinstance(output_config, dict): + output_format = output_config.get("format") if isinstance(output_format, dict) and output_format.get("type") == "json_schema": schema = output_format.get("schema") if schema: @@ -290,10 +326,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter: } } + # context_management: Anthropic dict -> OpenAI array + context_management = anthropic_request.get("context_management") + if isinstance(context_management, dict): + openai_cm = self.translate_context_management_to_responses_api(context_management) + if openai_cm is not None: + responses_kwargs["context_management"] = openai_cm + # metadata user_id -> user metadata = anthropic_request.get("metadata") if isinstance(metadata, dict) and "user_id" in metadata: - responses_kwargs["user"] = metadata["user_id"] + responses_kwargs["user"] = str(metadata["user_id"])[:64] return responses_kwargs From 6a68e3bba3ab5d5845139c6ae7073a5366bacef1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 18:14:06 +0530 Subject: [PATCH 08/54] Add tests for messages to responses transformation: --- .../responses_adapters/__init__.py | 0 .../test_responses_adapters_transformation.py | 987 ++++++++++++++++++ 2 files changed, 987 insertions(+) create mode 100644 tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py create mode 100644 tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py new file mode 100644 index 00000000000..252ba230ff7 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -0,0 +1,987 @@ +""" +Tests for LiteLLMAnthropicToResponsesAPIAdapter +(litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py) +""" + +import json +import os +import sys +from typing import Any, Dict, List +from unittest.mock import MagicMock + +sys.path.insert(0, os.path.abspath("../../../../../../..")) + +from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import ( + LiteLLMAnthropicToResponsesAPIAdapter, +) +from litellm.types.llms.anthropic import AnthropicMessagesRequest + + +def _make_request(**overrides) -> AnthropicMessagesRequest: + base: dict = { + "model": "openai.gpt-5.1-codex", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 1024, + } + base.update(overrides) + return AnthropicMessagesRequest(**base) + + +_ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter() + + +# --------------------------------------------------------------------------- +# context_management conversion +# --------------------------------------------------------------------------- + + +class TestContextManagementConversion: + """Anthropic dict -> OpenAI array conversion for context_management.""" + + def test_compact_edit_converted_to_array(self): + """compact_20260112 with trigger maps to OpenAI compaction entry.""" + cm = { + "edits": [ + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 150000}, + } + ] + } + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result == [{"type": "compaction", "compact_threshold": 150000}] + + def test_compact_edit_without_trigger(self): + """compact_20260112 without a trigger still maps to a compaction entry.""" + cm = {"edits": [{"type": "compact_20260112"}]} + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result == [{"type": "compaction"}] + + def test_unknown_edit_type_is_dropped(self): + """Anthropic-only edit types (e.g. clear_thinking) are silently dropped.""" + cm = {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]} + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result is None + + def test_mixed_edits_only_known_types_kept(self): + """Only compact_20260112 is converted; unknown types are dropped.""" + cm = { + "edits": [ + {"type": "clear_thinking_20251015", "keep": "all"}, + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 200000}, + }, + ] + } + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result == [{"type": "compaction", "compact_threshold": 200000}] + + def test_non_dict_returns_none(self): + result = _ADAPTER.translate_context_management_to_responses_api([]) # type: ignore + assert result is None + + def test_translate_request_includes_context_management(self): + """translate_request converts context_management and sets it on kwargs.""" + req = _make_request( + context_management={ + "edits": [ + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 100000}, + } + ] + } + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["context_management"] == [ + {"type": "compaction", "compact_threshold": 100000} + ] + + def test_translate_request_drops_anthropic_only_context_management(self): + """context_management with only unknown edit types is omitted from kwargs.""" + req = _make_request( + context_management={ + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + } + ) + kwargs = _ADAPTER.translate_request(req) + assert "context_management" not in kwargs + + +# --------------------------------------------------------------------------- +# structured output via output_config +# --------------------------------------------------------------------------- + + +class TestOutputConfigStructuredOutput: + """output_config.format.json_schema -> OpenAI text.format conversion.""" + + _SCHEMA = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string"}, + }, + "required": ["name", "email"], + "additionalProperties": False, + } + + def test_output_config_format_json_schema_converted(self): + """output_config.format.json_schema is converted to OpenAI text.format.""" + req = _make_request( + output_config={"format": {"type": "json_schema", "schema": self._SCHEMA}} + ) + kwargs = _ADAPTER.translate_request(req) + assert "text" in kwargs + fmt = kwargs["text"]["format"] + assert fmt["type"] == "json_schema" + assert fmt["schema"] == self._SCHEMA + assert fmt["strict"] is True + assert fmt["name"] == "structured_output" + + def test_output_config_without_format_does_not_set_text(self): + """output_config with only non-format keys doesn't produce text.format.""" + req = _make_request(output_config={"effort": "high"}) + kwargs = _ADAPTER.translate_request(req) + assert "text" not in kwargs + + def test_output_format_still_works(self): + """The original output_format field still takes precedence when present.""" + req = _make_request( + output_format={"type": "json_schema", "schema": self._SCHEMA} + ) + kwargs = _ADAPTER.translate_request(req) + assert "text" in kwargs + assert kwargs["text"]["format"]["type"] == "json_schema" + + def test_output_format_takes_precedence_over_output_config(self): + """output_format takes precedence over output_config.format.""" + other_schema = {"type": "object", "properties": {"id": {"type": "integer"}}} + req = _make_request( + output_format={"type": "json_schema", "schema": self._SCHEMA}, + output_config={"format": {"type": "json_schema", "schema": other_schema}}, + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["text"]["format"]["schema"] == self._SCHEMA + + +# --------------------------------------------------------------------------- +# translate_messages_to_responses_input +# --------------------------------------------------------------------------- + +# Helper: cast plain dicts to the expected type so call sites stay clean. +def _translate_messages(messages: List[Any]) -> List[Dict[str, Any]]: + return _ADAPTER.translate_messages_to_responses_input(messages) # type: ignore[arg-type] + + +class TestTranslateMessagesToResponsesInput: + """Anthropic messages list -> OpenAI Responses API input items.""" + + def test_user_string_content(self): + """Plain string user message becomes a message with input_text.""" + messages = [{"role": "user", "content": "Hello world"}] + result = _translate_messages(messages) + assert result == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Hello world"}], + } + ] + + def test_user_list_text_block(self): + """User message with text content block maps to input_text.""" + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "What is 2+2?"}], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "What is 2+2?"}], + } + ] + + def test_user_multiple_text_blocks(self): + """Multiple text blocks in a user message are all converted.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "First part."}, + {"type": "text", "text": "Second part."}, + ], + } + ] + result = _translate_messages(messages) + assert len(result) == 1 + assert result[0]["content"] == [ + {"type": "input_text", "text": "First part."}, + {"type": "input_text", "text": "Second part."}, + ] + + def test_user_base64_image(self): + """User message with base64 image source becomes input_image with data URL.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc123", + }, + } + ], + } + ] + result = _translate_messages(messages) + assert len(result) == 1 + assert result[0]["content"] == [ + {"type": "input_image", "image_url": "data:image/png;base64,abc123"} + ] + + def test_user_url_image(self): + """User message with URL image source becomes input_image with the URL.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/img.jpg"}, + } + ], + } + ] + result = _translate_messages(messages) + assert result[0]["content"] == [ + {"type": "input_image", "image_url": "https://example.com/img.jpg"} + ] + + def test_user_base64_image_empty_data_skipped(self): + """Base64 image with empty data is skipped (no URL can be formed).""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/jpeg", "data": ""}, + } + ], + } + ] + result = _translate_messages(messages) + # No user_parts -> no message item appended + assert result == [] + + def test_user_tool_result_string_content(self): + """tool_result with string content becomes function_call_output.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_abc", + "content": "42 degrees", + } + ], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "function_call_output", + "call_id": "call_abc", + "output": "42 degrees", + } + ] + + def test_user_tool_result_list_content(self): + """tool_result with list of text blocks is joined into a single string.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_xyz", + "content": [ + {"type": "text", "text": "Line 1"}, + {"type": "text", "text": "Line 2"}, + ], + } + ], + } + ] + result = _translate_messages(messages) + assert result[0]["output"] == "Line 1\nLine 2" + + def test_user_tool_result_null_content(self): + """tool_result with null content becomes empty string output.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "call_null", "content": None} + ], + } + ] + result = _translate_messages(messages) + assert result[0]["output"] == "" + + def test_assistant_string_content(self): + """Plain string assistant message becomes a message with output_text.""" + messages = [{"role": "assistant", "content": "I can help with that."}] + result = _translate_messages(messages) + assert result == [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "I can help with that."}], + } + ] + + def test_assistant_text_block(self): + """Assistant message with text block maps to output_text.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "text", "text": "Here is the answer."}], + } + ] + result = _translate_messages(messages) + assert result[0]["content"] == [ + {"type": "output_text", "text": "Here is the answer."} + ] + + def test_assistant_tool_use_becomes_function_call(self): + """Assistant tool_use block becomes a top-level function_call item.""" + messages = [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01", + "name": "get_weather", + "input": {"location": "Boston"}, + } + ], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "function_call", + "call_id": "toolu_01", + "name": "get_weather", + "arguments": json.dumps({"location": "Boston"}), + } + ] + + def test_assistant_thinking_block_becomes_output_text(self): + """Assistant thinking block text is included as output_text.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Let me reason step by step."} + ], + } + ] + result = _translate_messages(messages) + assert result[0]["content"] == [ + {"type": "output_text", "text": "Let me reason step by step."} + ] + + def test_assistant_empty_thinking_block_skipped(self): + """Assistant thinking block with empty thinking text is skipped.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "thinking", "thinking": ""}], + } + ] + result = _translate_messages(messages) + assert result == [] + + def test_mixed_messages_ordering(self): + """Full multi-turn conversation is converted in order.""" + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_02", + "name": "get_weather", + "input": {"city": "NYC"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_02", + "content": "Sunny, 72F", + } + ], + }, + {"role": "assistant", "content": "It's sunny and 72°F in NYC."}, + ] + result = _translate_messages(messages) + types = [item["type"] for item in result] + assert types == ["message", "function_call", "function_call_output", "message"] + + def test_user_text_and_image_mixed(self): + """User message with both text and image produces both parts.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image:"}, + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/cat.jpg"}, + }, + ], + } + ] + result = _translate_messages(messages) + assert len(result) == 1 + assert result[0]["content"][0] == {"type": "input_text", "text": "Describe this image:"} + assert result[0]["content"][1] == { + "type": "input_image", + "image_url": "https://example.com/cat.jpg", + } + + def test_unknown_image_source_type_skipped(self): + """Image block with unknown source type is silently skipped.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"type": "file_path", "path": "/tmp/img.png"}, + } + ], + } + ] + result = _translate_messages(messages) + assert result == [] + + +# --------------------------------------------------------------------------- +# translate_tools_to_responses_api +# --------------------------------------------------------------------------- + + +class TestTranslateToolsToResponsesAPI: + """Anthropic tool definitions -> Responses API function tools.""" + + def test_regular_tool_with_description_and_schema(self): + """Standard tool with description and input_schema is converted to function.""" + tools = [ + { + "name": "get_weather", + "description": "Get current weather for a city.", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result == [ + { + "type": "function", + "name": "get_weather", + "description": "Get current weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ] + + def test_tool_without_description(self): + """Tool without a description omits the description key.""" + tools = [{"name": "ping", "input_schema": {"type": "object", "properties": {}}}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result[0]["type"] == "function" + assert result[0]["name"] == "ping" + assert "description" not in result[0] + + def test_tool_without_input_schema(self): + """Tool without input_schema omits the parameters key.""" + tools = [{"name": "no_schema_tool", "description": "Does something."}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result[0]["type"] == "function" + assert "parameters" not in result[0] + + def test_web_search_tool_by_name(self): + """Tool named 'web_search' maps to web_search_preview.""" + tools = [{"name": "web_search", "type": "custom"}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result == [{"type": "web_search_preview"}] + + def test_web_search_tool_by_type_prefix(self): + """Tool with type starting with 'web_search' maps to web_search_preview.""" + tools = [{"name": "search", "type": "web_search_20250305"}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result == [{"type": "web_search_preview"}] + + def test_multiple_tools_order_preserved(self): + """Multiple tools are converted in order.""" + tools = [ + {"name": "tool_a", "description": "A"}, + {"name": "web_search", "type": "custom"}, + {"name": "tool_b", "description": "B"}, + ] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert len(result) == 3 + assert result[0]["name"] == "tool_a" + assert result[1] == {"type": "web_search_preview"} + assert result[2]["name"] == "tool_b" + + def test_empty_tools_list(self): + """Empty tools list returns empty list.""" + assert _ADAPTER.translate_tools_to_responses_api([]) == [] + + +# --------------------------------------------------------------------------- +# translate_tool_choice_to_responses_api +# --------------------------------------------------------------------------- + + +class TestTranslateToolChoiceToResponsesAPI: + """Anthropic tool_choice -> Responses API tool_choice.""" + + def test_auto_maps_to_auto(self): + assert _ADAPTER.translate_tool_choice_to_responses_api({"type": "auto"}) == { + "type": "auto" + } + + def test_any_maps_to_required(self): + assert _ADAPTER.translate_tool_choice_to_responses_api({"type": "any"}) == { + "type": "required" + } + + def test_specific_tool_maps_to_function(self): + result = _ADAPTER.translate_tool_choice_to_responses_api( + {"type": "tool", "name": "get_weather"} + ) + assert result == {"type": "function", "name": "get_weather"} + + def test_unknown_type_defaults_to_auto(self): + result = _ADAPTER.translate_tool_choice_to_responses_api({"type": "none"}) + assert result == {"type": "auto"} + + +# --------------------------------------------------------------------------- +# translate_thinking_to_reasoning +# --------------------------------------------------------------------------- + + +class TestTranslateThinkingToReasoning: + """Anthropic thinking param -> Responses API reasoning param.""" + + def test_budget_high_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 10000} + ) + assert result == {"effort": "high", "summary": "detailed"} + + def test_budget_above_threshold_high_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 50000} + ) + assert result is not None + assert result["effort"] == "high" + + def test_budget_medium_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 7500} + ) + assert result == {"effort": "medium", "summary": "detailed"} + + def test_budget_low_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 3000} + ) + assert result == {"effort": "low", "summary": "detailed"} + + def test_budget_minimal_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 500} + ) + assert result == {"effort": "minimal", "summary": "detailed"} + + def test_budget_at_exact_thresholds(self): + result_medium = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 5000} + ) + assert result_medium is not None + assert result_medium["effort"] == "medium" + result_low = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 2000} + ) + assert result_low is not None + assert result_low["effort"] == "low" + + def test_disabled_type_returns_none(self): + result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}) + assert result is None + + def test_non_dict_returns_none(self): + result = _ADAPTER.translate_thinking_to_reasoning("enabled") # type: ignore + assert result is None + + def test_missing_budget_defaults_to_minimal(self): + """Missing budget_tokens defaults to 0, which is < 2000 -> minimal.""" + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}) + assert result == {"effort": "minimal", "summary": "detailed"} + + +# --------------------------------------------------------------------------- +# translate_request – broader coverage +# --------------------------------------------------------------------------- + + +class TestTranslateRequestBroaderCoverage: + """Full translate_request call: field-by-field mapping verification.""" + + def test_model_and_input_always_present(self): + req = _make_request() + kwargs = _ADAPTER.translate_request(req) + assert "model" in kwargs + assert "input" in kwargs + + def test_system_string_becomes_instructions(self): + req = _make_request(system="You are a helpful assistant.") + kwargs = _ADAPTER.translate_request(req) + assert kwargs["instructions"] == "You are a helpful assistant." + + def test_system_list_of_text_blocks_joined(self): + req = _make_request( + system=[ + {"type": "text", "text": "Be concise."}, + {"type": "text", "text": "Be helpful."}, + ] + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["instructions"] == "Be concise.\nBe helpful." + + def test_system_list_skips_non_text_blocks(self): + req = _make_request( + system=[ + {"type": "image", "source": {}}, + {"type": "text", "text": "Only text matters."}, + ] + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["instructions"] == "Only text matters." + + def test_max_tokens_mapped_to_max_output_tokens(self): + req = _make_request(max_tokens=512) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["max_output_tokens"] == 512 + + def test_temperature_passed_through(self): + req = _make_request(temperature=0.7) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["temperature"] == 0.7 + + def test_top_p_passed_through(self): + req = _make_request(top_p=0.9) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["top_p"] == 0.9 + + def test_tools_translated(self): + req = _make_request( + tools=[{"name": "calculator", "description": "Does math.", "input_schema": {}}] + ) + kwargs = _ADAPTER.translate_request(req) + assert len(kwargs["tools"]) == 1 + assert kwargs["tools"][0]["name"] == "calculator" + + def test_tool_choice_translated(self): + req = _make_request( + tools=[{"name": "do_thing"}], + tool_choice={"type": "tool", "name": "do_thing"}, + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["tool_choice"] == {"type": "function", "name": "do_thing"} + + def test_thinking_translated_to_reasoning(self): + req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["reasoning"] == {"effort": "high", "summary": "detailed"} + + def test_disabled_thinking_not_included_in_kwargs(self): + req = _make_request(thinking={"type": "disabled"}) + kwargs = _ADAPTER.translate_request(req) + assert "reasoning" not in kwargs + + def test_metadata_user_id_mapped_to_user(self): + req = _make_request(metadata={"user_id": "user-42"}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["user"] == "user-42" + + def test_metadata_user_id_truncated_to_64_chars(self): + long_id = "x" * 100 + req = _make_request(metadata={"user_id": long_id}) + kwargs = _ADAPTER.translate_request(req) + assert len(kwargs["user"]) == 64 + + def test_no_optional_fields_does_not_add_spurious_keys(self): + req = _make_request() + kwargs = _ADAPTER.translate_request(req) + for key in ("instructions", "temperature", "top_p", "tools", "tool_choice", + "reasoning", "text", "context_management", "user"): + assert key not in kwargs, f"unexpected key: {key}" + + +# --------------------------------------------------------------------------- +# translate_response +# --------------------------------------------------------------------------- + + +def _make_mock_response( + output: list, + status: str = "completed", + response_id: str = "resp_001", + model: str = "gpt-4o", + input_tokens: int = 100, + output_tokens: int = 50, +) -> MagicMock: + """Build a minimal mock ResponsesAPIResponse.""" + usage = MagicMock() + usage.input_tokens = input_tokens + usage.output_tokens = output_tokens + + resp = MagicMock() + resp.id = response_id + resp.model = model + resp.status = status + resp.output = output + resp.usage = usage + return resp + + +def _make_output_message(texts: List[str]) -> MagicMock: + """Build a mock ResponseOutputMessage with output_text parts.""" + from openai.types.responses import ResponseOutputMessage # type: ignore[import] + + parts = [] + for t in texts: + part = MagicMock() + part.type = "output_text" + part.text = t + parts.append(part) + + msg = MagicMock(spec=ResponseOutputMessage) + msg.content = parts + return msg + + +def _make_function_call_item( + call_id: str, name: str, arguments: str +) -> MagicMock: + """Build a mock ResponseFunctionToolCall.""" + from openai.types.responses import ResponseFunctionToolCall # type: ignore[import] + + item = MagicMock(spec=ResponseFunctionToolCall) + item.call_id = call_id + item.id = call_id + item.name = name + item.arguments = arguments + return item + + +def _make_reasoning_item(summaries: List[str]) -> MagicMock: + """Build a mock ResponseReasoningItem.""" + from openai.types.responses import ResponseReasoningItem # type: ignore[import] + + summary_mocks = [] + for text in summaries: + s = MagicMock() + s.text = text + summary_mocks.append(s) + + item = MagicMock(spec=ResponseReasoningItem) + item.summary = summary_mocks + return item + + +class TestTranslateResponse: + """Responses API -> AnthropicMessagesResponse conversion.""" + + def test_output_text_message_becomes_text_block(self): + """ResponseOutputMessage with output_text parts -> Anthropic text content.""" + response = _make_mock_response(output=[_make_output_message(["Hello!"])]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Hello!" + + def test_multiple_text_parts(self): + """Multiple output_text parts become multiple text content blocks.""" + response = _make_mock_response( + output=[_make_output_message(["Part 1", "Part 2"])] + ) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 2 + assert result["content"][0]["text"] == "Part 1" + assert result["content"][1]["text"] == "Part 2" + + def test_function_call_becomes_tool_use(self): + """ResponseFunctionToolCall -> Anthropic tool_use content block.""" + fc = _make_function_call_item("call_99", "get_weather", '{"city": "NYC"}') + response = _make_mock_response(output=[fc]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + block = result["content"][0] + assert block["type"] == "tool_use" + assert block["id"] == "call_99" + assert block["name"] == "get_weather" + assert block["input"] == {"city": "NYC"} + + def test_function_call_sets_stop_reason_tool_use(self): + """Presence of a function_call sets stop_reason to 'tool_use'.""" + fc = _make_function_call_item("call_1", "tool_a", "{}") + response = _make_mock_response(output=[fc]) + result: Any = _ADAPTER.translate_response(response) + assert result["stop_reason"] == "tool_use" + + def test_text_only_stop_reason_end_turn(self): + """Text-only response has stop_reason 'end_turn'.""" + response = _make_mock_response(output=[_make_output_message(["Hi"])]) + result: Any = _ADAPTER.translate_response(response) + assert result["stop_reason"] == "end_turn" + + def test_incomplete_status_sets_max_tokens(self): + """status='incomplete' overrides stop_reason to 'max_tokens'.""" + response = _make_mock_response( + output=[_make_output_message(["Truncated..."])], + status="incomplete", + ) + result: Any = _ADAPTER.translate_response(response) + assert result["stop_reason"] == "max_tokens" + + def test_reasoning_item_becomes_thinking_block(self): + """ResponseReasoningItem summaries -> Anthropic thinking content blocks.""" + reasoning = _make_reasoning_item(["Step 1: analyze. Step 2: conclude."]) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "thinking" + assert "Step 1" in result["content"][0]["thinking"] + + def test_empty_reasoning_summary_skipped(self): + """Reasoning item with empty text summary is not added to content.""" + reasoning = _make_reasoning_item([""]) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [] + + def test_usage_mapped_correctly(self): + """Input/output tokens from ResponseAPIUsage are mapped to AnthropicUsage.""" + response = _make_mock_response( + output=[_make_output_message(["OK"])], + input_tokens=200, + output_tokens=75, + ) + result: Any = _ADAPTER.translate_response(response) + assert result["usage"]["input_tokens"] == 200 + assert result["usage"]["output_tokens"] == 75 + + def test_model_and_id_preserved(self): + """Model and response ID from the Responses API are forwarded.""" + response = _make_mock_response( + output=[_make_output_message(["Hi"])], + response_id="resp_xyz", + model="gpt-4-turbo", + ) + result: Any = _ADAPTER.translate_response(response) + assert result["id"] == "resp_xyz" + assert result["model"] == "gpt-4-turbo" + + def test_role_is_always_assistant(self): + response = _make_mock_response(output=[_make_output_message(["Hi"])]) + result: Any = _ADAPTER.translate_response(response) + assert result["role"] == "assistant" + + def test_type_is_always_message(self): + response = _make_mock_response(output=[_make_output_message(["Hi"])]) + result: Any = _ADAPTER.translate_response(response) + assert result["type"] == "message" + + def test_empty_output_list(self): + """Empty output list produces empty content with 'end_turn' stop reason.""" + response = _make_mock_response(output=[]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [] + assert result["stop_reason"] == "end_turn" + + def test_function_call_with_invalid_json_arguments(self): + """Invalid JSON in function_call arguments falls back to empty dict.""" + fc = _make_function_call_item("call_bad", "broken_tool", "not-valid-json") + response = _make_mock_response(output=[fc]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["input"] == {} + + def test_dict_output_message_item(self): + """Dict-shaped output message (type=message) is also handled.""" + output_item = { + "type": "message", + "content": [{"type": "output_text", "text": "Dict-based response"}], + } + response = _make_mock_response(output=[output_item]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Dict-based response" + + def test_dict_function_call_item(self): + """Dict-shaped function_call item is converted to tool_use block.""" + output_item = { + "type": "function_call", + "call_id": "call_dict_1", + "name": "search", + "arguments": '{"query": "cats"}', + } + response = _make_mock_response(output=[output_item]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["type"] == "tool_use" + assert result["content"][0]["name"] == "search" + assert result["content"][0]["input"] == {"query": "cats"} + assert result["stop_reason"] == "tool_use" + + def test_mixed_reasoning_text_and_tool_use(self): + """Reasoning + text + tool_use in one response all convert correctly.""" + reasoning = _make_reasoning_item(["Thinking..."]) + text_msg = _make_output_message(["Here is my answer."]) + fc = _make_function_call_item("call_mix", "lookup", '{"id": 1}') + response = _make_mock_response(output=[reasoning, text_msg, fc]) + result: Any = _ADAPTER.translate_response(response) + types = [b["type"] for b in result["content"]] + assert "thinking" in types + assert "text" in types + assert "tool_use" in types + assert result["stop_reason"] == "tool_use" From d3d11fb06ec624b4abc6f3ae2d2e950cc59a6eac Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Fri, 27 Feb 2026 01:07:18 +0530 Subject: [PATCH 09/54] feat: add tags in project --- litellm/proxy/_types.py | 25 ++++++++++++- litellm/proxy/auth/user_api_key_auth.py | 12 +++++- litellm/proxy/litellm_pre_call_utils.py | 24 +++++++++--- .../management_endpoints/project_endpoints.py | 20 ++++++++++ .../test_project_tags_pydantic.py | 37 +++++++++++++++++++ 5 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 tests/test_litellm/test_project_tags_pydantic.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 53513f7f522..8713f54a3ef 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2280,6 +2280,9 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): organization_rpm_limit: Optional[int] = None organization_metadata: Optional[dict] = None + # Project Params + project_metadata: Optional[dict] = None + # Time stamps last_refreshed_at: Optional[float] = None # last time joint view was pulled from db @@ -2581,6 +2584,7 @@ class NewProjectRequest(LiteLLM_BudgetTable): team_id: str budget_id: Optional[str] = None metadata: Optional[dict] = None + tags: Optional[List[str]] = None models: List[str] = [] model_rpm_limit: Optional[dict] = None model_tpm_limit: Optional[dict] = None @@ -2590,7 +2594,15 @@ class NewProjectRequest(LiteLLM_BudgetTable): @model_validator(mode="before") @classmethod def set_model_info(cls, values): - for field in LiteLLM_ManagementEndpoint_MetadataFields: + if "tags" in values and values["tags"] is not None: + if not isinstance(values["tags"], list): + raise ValueError( + f"tags must be a list of strings, got {type(values['tags']).__name__}" + ) + for field in ( + LiteLLM_ManagementEndpoint_MetadataFields + + LiteLLM_ManagementEndpoint_MetadataFields_Premium + ): if values.get(field) is not None: if values.get("metadata") is None: values.update({"metadata": {}}) @@ -2607,6 +2619,7 @@ class UpdateProjectRequest(LiteLLM_BudgetTable): description: Optional[str] = None team_id: Optional[str] = None metadata: Optional[dict] = None + tags: Optional[List[str]] = None models: Optional[List[str]] = None model_rpm_limit: Optional[dict] = None model_tpm_limit: Optional[dict] = None @@ -2617,7 +2630,15 @@ class UpdateProjectRequest(LiteLLM_BudgetTable): @model_validator(mode="before") @classmethod def set_model_info(cls, values): - for field in LiteLLM_ManagementEndpoint_MetadataFields: + if "tags" in values and values["tags"] is not None: + if not isinstance(values["tags"], list): + raise ValueError( + f"tags must be a list of strings, got {type(values['tags']).__name__}" + ) + for field in ( + LiteLLM_ManagementEndpoint_MetadataFields + + LiteLLM_ManagementEndpoint_MetadataFields_Premium + ): if values.get(field) is not None: if values.get("metadata") is None: values.update({"metadata": {}}) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 3e2378ada60..8ad3b83c043 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -212,10 +212,12 @@ async def user_api_key_auth_websocket(websocket: WebSocket): api_key = websocket.headers.get("api-key") if not api_key: # Try extracting from WebSocket subprotocol (browser clients) - for protocol in websocket.headers.get("sec-websocket-protocol", "").split(","): + for protocol in websocket.headers.get("sec-websocket-protocol", "").split( + "," + ): protocol = protocol.strip() if protocol.startswith("openai-insecure-api-key."): - api_key = protocol[len("openai-insecure-api-key."):] + api_key = protocol[len("openai-insecure-api-key.") :] break if not api_key: await websocket.close(code=status.WS_1008_POLICY_VIOLATION) @@ -704,6 +706,8 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + if _jwt_project_obj is not None: + valid_token.project_metadata = _jwt_project_obj.metadata # run through common checks _ = await common_checks( @@ -1294,6 +1298,8 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + if _project_obj is not None: + valid_token.project_metadata = _project_obj.metadata global_proxy_spend = None if ( @@ -1743,6 +1749,8 @@ async def _run_post_custom_auth_checks( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + if _project_obj is not None: + valid_token.project_metadata = _project_obj.metadata _ = await common_checks( request=request, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 7ebf9a4caee..eac1c4a33c6 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -248,13 +248,15 @@ def clean_headers( clean_headers = {} litellm_key_lower = ( litellm_key_header_name.lower() if litellm_key_header_name is not None else None - ) + ) for header, value in headers.items(): header_lower = header.lower() - + if header_lower == "authorization" and is_anthropic_oauth_key(value): clean_headers[header] = value - elif forward_llm_provider_auth_headers and header_lower in _SPECIAL_HEADERS_CACHE: + elif ( + forward_llm_provider_auth_headers and header_lower in _SPECIAL_HEADERS_CACHE + ): if litellm_key_lower and header_lower == litellm_key_lower: continue if header_lower == "authorization": @@ -840,11 +842,13 @@ async def add_litellm_data_to_request( # noqa: PLR0915 from litellm.types.proxy.litellm_pre_call_utils import SecretFields _raw_headers: Dict[str, str] = _safe_get_request_headers(request) - + forward_llm_auth = False if general_settings: - forward_llm_auth = general_settings.get("forward_llm_provider_auth_headers", False) - + forward_llm_auth = general_settings.get( + "forward_llm_provider_auth_headers", False + ) + _headers: Dict[str, str] = clean_headers( request.headers, litellm_key_header_name=( @@ -1019,6 +1023,14 @@ async def add_litellm_data_to_request( # noqa: PLR0915 "spend_logs_metadata" ] + ## PROJECT-LEVEL SPEND LOGS/TAGS + project_metadata = user_api_key_dict.project_metadata or {} + if "tags" in project_metadata and project_metadata["tags"] is not None: + data[_metadata_variable_name]["tags"] = LiteLLMProxyRequestSetup._merge_tags( + request_tags=data[_metadata_variable_name].get("tags"), + tags_to_add=project_metadata["tags"], + ) + ## TEAM-LEVEL METADATA data = ( LiteLLMProxyRequestSetup.add_management_endpoint_metadata_to_request_metadata( diff --git a/litellm/proxy/management_endpoints/project_endpoints.py b/litellm/proxy/management_endpoints/project_endpoints.py index ba3238ebfd5..c825802f314 100644 --- a/litellm/proxy/management_endpoints/project_endpoints.py +++ b/litellm/proxy/management_endpoints/project_endpoints.py @@ -284,6 +284,7 @@ async def new_project( - model_tpm_limit: *Optional[dict]* - TPM limits per model. Example: {"gpt-4": 50000, "gpt-3.5-turbo": 100000} - budget_duration: *Optional[str]* - Frequency of reseting project budget - metadata: *Optional[dict]* - Metadata for project, store information for project. Example metadata - {"use_case_id": "SNOW-12345", "responsible_ai_id": "RAI-67890"} + - tags: *Optional[list]* - Tags for the project. Example: ["production", "api"] - blocked: *bool* - Flag indicating if the project is blocked or not - will stop all calls from keys with this project_id. - object_permission: Optional[LiteLLM_ObjectPermissionBase] - project-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"]}. IF null or {} then no object permission. @@ -339,6 +340,15 @@ async def new_project( ) try: + if getattr(data, "tags", None) is not None and not premium_user: + raise HTTPException( + status_code=403, + detail={ + "error": "Only premium users can add tags to projects. " + + CommonProxyErrors.not_premium_user.value + }, + ) + if not premium_user: raise HTTPException( status_code=403, @@ -485,6 +495,7 @@ async def update_project( - model_rpm_limit: *Optional[dict]* - Updated RPM limits per model - model_tpm_limit: *Optional[dict]* - Updated TPM limits per model - budget_duration: *Optional[str]* - Updated budget duration + - tags: *Optional[list]* - Updated list of tags for the project - object_permission: Optional[LiteLLM_ObjectPermissionBase] - Updated object permission Example: @@ -514,6 +525,15 @@ async def update_project( ) try: + if getattr(data, "tags", None) is not None and not premium_user: + raise HTTPException( + status_code=403, + detail={ + "error": "Only premium users can add tags to projects. " + + CommonProxyErrors.not_premium_user.value + }, + ) + if not premium_user: raise HTTPException( status_code=403, diff --git a/tests/test_litellm/test_project_tags_pydantic.py b/tests/test_litellm/test_project_tags_pydantic.py new file mode 100644 index 00000000000..ed3b29fe7be --- /dev/null +++ b/tests/test_litellm/test_project_tags_pydantic.py @@ -0,0 +1,37 @@ +import pytest +from litellm.proxy._types import NewProjectRequest, UpdateProjectRequest + + +def test_new_project_request_tags(): + # Test tags are correctly moved to metadata["tags"] + req = NewProjectRequest( + project_id="test_proj", team_id="team_1", tags=["tag1", "tag2"] + ) + + # After validation, tags should be inside metadata + assert req.metadata is not None + assert "tags" in req.metadata + assert req.metadata["tags"] == ["tag1", "tag2"] + assert req.tags is None # Or removed dependending on pydantic version + + +def test_update_project_request_tags(): + # Test tags are correctly moved to metadata["tags"] + req = UpdateProjectRequest(project_id="test_proj", tags=["new_tag"]) + + assert req.metadata is not None + assert "tags" in req.metadata + assert req.metadata["tags"] == ["new_tag"] + assert req.tags is None + + +def test_new_project_request_invalid_tags_type(): + # tags must be a list — a string should raise a ValidationError + with pytest.raises(Exception): + NewProjectRequest(project_id="test_proj", team_id="team_1", tags="not-a-list") + + +def test_update_project_request_invalid_tags_type(): + # tags must be a list — a string should raise a ValidationError + with pytest.raises(Exception): + UpdateProjectRequest(project_id="test_proj", tags="not-a-list") From f24a41898bba81bdb047e84fd66c1c9bc0f09832 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Thu, 26 Feb 2026 14:41:59 -0600 Subject: [PATCH 10/54] feat(vertex): add gemini-3.1-flash-image-preview model DB support - add gemini-3.1-flash-image-preview + vertex_ai alias entries\n- set pricing to Gemini 3.1 Flash Image Preview rates\n- mirror updates in packaged backup model map\n- update llm cost calc regression test to cover new model --- ...odel_prices_and_context_window_backup.json | 45 +++++++++++++++++++ model_prices_and_context_window.json | 45 +++++++++++++++++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 27 ++++++----- 3 files changed, 106 insertions(+), 11 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 57563fc0bcc..b21f23ac022 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14194,6 +14194,38 @@ "supports_vision": true, "supports_web_search": true }, + "gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true + }, "deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -31545,6 +31577,19 @@ "output_cost_per_token_batches": 6e-06, "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, + "vertex_ai/gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" + }, "vertex_ai/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 57563fc0bcc..b21f23ac022 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14194,6 +14194,38 @@ "supports_vision": true, "supports_web_search": true }, + "gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true + }, "deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -31545,6 +31577,19 @@ "output_cost_per_token_batches": 6e-06, "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, + "vertex_ai/gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" + }, "vertex_ai/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index b45cbbd99c0..9abce33fcd0 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -766,7 +766,14 @@ def test_service_tier_fallback_pricing(): assert abs(std_cost[1] - expected_standard_completion) < 1e-10, f"Standard completion cost mismatch: {std_cost[1]} vs {expected_standard_completion}" -def test_gemini_image_generation_cost_with_zero_text_tokens(): +@pytest.mark.parametrize( + "model", + [ + "gemini-3-pro-image-preview", + "gemini-3.1-flash-image-preview", + ], +) +def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): """ Test that image_tokens are correctly costed when text_tokens=0. @@ -779,7 +786,6 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini-3-pro-image-preview" custom_llm_provider = "vertex_ai" # Usage from the issue: text_tokens=0, image_tokens=1120, reasoning_tokens=225 @@ -809,9 +815,9 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(): # Expected costs: # - text_tokens: 0 * output_cost_per_token = 0 - # - image_tokens: 1120 * output_cost_per_image_token = 1120 * 1.2e-04 = 0.1344 - # - reasoning_tokens: 225 * output_cost_per_token = 225 * 1.2e-05 = 0.0027 - # Total completion: ~0.1371 + # - image_tokens: 1120 * output_cost_per_image_token + # - reasoning_tokens: 225 * output_cost_per_token + # Total completion should include both image + reasoning costs. output_cost_per_image_token = model_cost_map.get("output_cost_per_image_token", 0) output_cost_per_token = model_cost_map.get("output_cost_per_token", 0) @@ -820,12 +826,11 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(): expected_reasoning_cost = 225 * output_cost_per_token # reasoning uses base token cost expected_completion_cost = expected_image_cost + expected_reasoning_cost - # The bug was: all 1345 tokens were treated as text = 1345 * 1.2e-05 = 0.01614 - # Fixed: image_tokens use image pricing = ~0.137 - - assert completion_cost > 0.10, ( - f"Completion cost should be > $0.10 (image tokens are expensive), got ${completion_cost:.6f}. " - f"Bug: tokens may be incorrectly treated as text tokens." + # The bug was: all completion tokens were treated as text tokens only. + bugged_text_only_cost = 1345 * output_cost_per_token + assert completion_cost > bugged_text_only_cost * 2, ( + f"Completion cost should be significantly larger than text-only bugged path. " + f"Expected > {bugged_text_only_cost * 2:.6f}, got {completion_cost:.6f}" ) assert round(completion_cost, 4) == round(expected_completion_cost, 4), ( f"Expected completion cost ${expected_completion_cost:.6f}, got ${completion_cost:.6f}" From 702d5e88b8616fd1d13082be725612c482073568 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Thu, 26 Feb 2026 14:57:11 -0600 Subject: [PATCH 11/54] fix(cost): use token usage for gemini/vertex image generation when available - compute image_generation cost from usage token metadata for vertex/gemini\n- map ImageUsage to Usage and reuse generic_cost_per_token\n- fallback to output_cost_per_image when usage metadata missing\n- add tests for token-based path and fallback path --- .../image_generation/cost_calculator.py | 72 ++++++++- .../image_generation/cost_calculator.py | 71 ++++++++- .../llm_cost_calc/test_llm_cost_calc_utils.py | 144 ++++++++++++++++++ 3 files changed, 284 insertions(+), 3 deletions(-) diff --git a/litellm/llms/gemini/image_generation/cost_calculator.py b/litellm/llms/gemini/image_generation/cost_calculator.py index 0a9ca2e5276..6d7572d5522 100644 --- a/litellm/llms/gemini/image_generation/cost_calculator.py +++ b/litellm/llms/gemini/image_generation/cost_calculator.py @@ -2,10 +2,71 @@ Google AI Image Generation Cost Calculator """ -from typing import Any +from typing import Any, Optional import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.types.utils import ( + CompletionTokensDetailsWrapper, + ImageResponse, + PromptTokensDetailsWrapper, + Usage, +) + + +def _calculate_token_based_cost(model: str, image_response: ImageResponse) -> Optional[float]: + """ + Calculate token-based image generation cost when usage metadata is available. + + Falls back to None when usage metadata is missing/incomplete. + """ + usage = image_response.usage + if usage is None: + return None + + prompt_tokens = usage.input_tokens + completion_tokens = usage.output_tokens + total_tokens = usage.total_tokens + + if ( + prompt_tokens is None + or completion_tokens is None + or total_tokens is None + ): + return None + # ImageResponse may carry a default zeroed usage object even when provider + # usage metadata is absent. Treat this as missing usage and fall back. + if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: + return None + + input_tokens_details = getattr(usage, "input_tokens_details", None) + prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None + if input_tokens_details is not None: + prompt_tokens_details = PromptTokensDetailsWrapper( + text_tokens=getattr(input_tokens_details, "text_tokens", None), + image_tokens=getattr(input_tokens_details, "image_tokens", None), + cached_tokens=0, + ) + + normalized_usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + prompt_tokens_details=prompt_tokens_details, + completion_tokens_details=CompletionTokensDetailsWrapper( + text_tokens=0, + image_tokens=completion_tokens, + reasoning_tokens=0, + audio_tokens=0, + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=normalized_usage, + custom_llm_provider="gemini", + ) + return prompt_cost + completion_cost def cost_calculator( @@ -20,6 +81,13 @@ def cost_calculator( custom_llm_provider="gemini", ) + if isinstance(image_response, ImageResponse): + token_based_cost = _calculate_token_based_cost( + model=model, image_response=image_response + ) + if token_based_cost is not None: + return token_based_cost + output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if isinstance(image_response, ImageResponse): diff --git a/litellm/llms/vertex_ai/image_generation/cost_calculator.py b/litellm/llms/vertex_ai/image_generation/cost_calculator.py index 646c6080a2e..ac587182f04 100644 --- a/litellm/llms/vertex_ai/image_generation/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_generation/cost_calculator.py @@ -2,8 +2,71 @@ Vertex AI Image Generation Cost Calculator """ +from typing import Optional + import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.types.utils import ( + CompletionTokensDetailsWrapper, + ImageResponse, + PromptTokensDetailsWrapper, + Usage, +) + + +def _calculate_token_based_cost(model: str, image_response: ImageResponse) -> Optional[float]: + """ + Calculate token-based image generation cost when usage metadata is available. + + Falls back to None when usage metadata is missing/incomplete. + """ + usage = image_response.usage + if usage is None: + return None + + prompt_tokens = usage.input_tokens + completion_tokens = usage.output_tokens + total_tokens = usage.total_tokens + + if ( + prompt_tokens is None + or completion_tokens is None + or total_tokens is None + ): + return None + # ImageResponse may carry a default zeroed usage object even when provider + # usage metadata is absent. Treat this as missing usage and fall back. + if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: + return None + + input_tokens_details = getattr(usage, "input_tokens_details", None) + prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None + if input_tokens_details is not None: + prompt_tokens_details = PromptTokensDetailsWrapper( + text_tokens=getattr(input_tokens_details, "text_tokens", None), + image_tokens=getattr(input_tokens_details, "image_tokens", None), + cached_tokens=0, + ) + + normalized_usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + prompt_tokens_details=prompt_tokens_details, + completion_tokens_details=CompletionTokensDetailsWrapper( + text_tokens=0, + image_tokens=completion_tokens, + reasoning_tokens=0, + audio_tokens=0, + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=normalized_usage, + custom_llm_provider="vertex_ai", + ) + return prompt_cost + completion_cost def cost_calculator( @@ -18,6 +81,12 @@ def cost_calculator( custom_llm_provider="vertex_ai", ) + token_based_cost = _calculate_token_based_cost( + model=model, image_response=image_response + ) + if token_based_cost is not None: + return token_based_cost + output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if image_response.data: diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 9abce33fcd0..7e8848be301 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -9,9 +9,19 @@ import litellm from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) +from litellm.llms.gemini.image_generation.cost_calculator import ( + cost_calculator as gemini_image_generation_cost_calculator, +) +from litellm.llms.vertex_ai.image_generation.cost_calculator import ( + cost_calculator as vertex_image_generation_cost_calculator, +) from litellm.types.llms.openai import FileSearchTool, WebSearchOptions from litellm.types.utils import ( CompletionTokensDetailsWrapper, + ImageObject, + ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, ModelInfo, ModelResponse, PromptTokensDetailsWrapper, @@ -837,6 +847,140 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): ) +def test_vertex_image_generation_cost_prefers_token_usage_metadata(): + """ + When usage metadata exists on image responses, Vertex image generation cost + should be calculated from token pricing, not flat output_cost_per_image. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini-3.1-flash-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") + + input_text_tokens = 50 + input_image_tokens = 1120 + output_image_tokens = 1120 + prompt_tokens = input_text_tokens + input_image_tokens + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")], + usage=ImageUsage( + input_tokens=prompt_tokens, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=input_text_tokens, + image_tokens=input_image_tokens, + ), + output_tokens=output_image_tokens, + total_tokens=prompt_tokens + output_image_tokens, + ), + ) + + cost = vertex_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_prompt_cost = prompt_tokens * model_info["input_cost_per_token"] + expected_completion_cost = output_image_tokens * model_info["output_cost_per_image_token"] + expected_total_cost = expected_prompt_cost + expected_completion_cost + + assert round(cost, 10) == round(expected_total_cost, 10) + # Ensure this is not falling back to flat per-image pricing. + assert cost != len(image_response.data) * model_info["output_cost_per_image"] + + +def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(): + """ + Without usage metadata, Vertex image generation cost should fall back to + output_cost_per_image * number_of_images. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini-3.1-flash-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] + ) + + cost = vertex_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_cost = len(image_response.data) * model_info["output_cost_per_image"] + assert round(cost, 10) == round(expected_cost, 10) + + +def test_gemini_image_generation_cost_prefers_token_usage_metadata(): + """ + When usage metadata exists on image responses, Gemini image generation cost + should be calculated from token pricing, not flat output_cost_per_image. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini/gemini-3-pro-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + input_text_tokens = 20 + input_image_tokens = 1120 + output_image_tokens = 1120 + prompt_tokens = input_text_tokens + input_image_tokens + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")], + usage=ImageUsage( + input_tokens=prompt_tokens, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=input_text_tokens, + image_tokens=input_image_tokens, + ), + output_tokens=output_image_tokens, + total_tokens=prompt_tokens + output_image_tokens, + ), + ) + + cost = gemini_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_prompt_cost = prompt_tokens * model_info["input_cost_per_token"] + expected_completion_cost = output_image_tokens * model_info["output_cost_per_image_token"] + expected_total_cost = expected_prompt_cost + expected_completion_cost + + assert round(cost, 10) == round(expected_total_cost, 10) + # Ensure this is not falling back to flat per-image pricing. + assert cost != len(image_response.data) * model_info["output_cost_per_image"] + + +def test_gemini_image_generation_cost_falls_back_to_flat_image_pricing(): + """ + Without usage metadata, Gemini image generation cost should fall back to + output_cost_per_image * number_of_images. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini/gemini-3-pro-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] + ) + + cost = gemini_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_cost = len(image_response.data) * model_info["output_cost_per_image"] + assert round(cost, 10) == round(expected_cost, 10) + + def test_bedrock_anthropic_prompt_caching(): """Test Bedrock Anthropic models with prompt caching return correct costs.""" model = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" From 7df61d4b92163f63884b24f6753894364e1810cf Mon Sep 17 00:00:00 2001 From: Dharamendra Kumar Date: Thu, 26 Feb 2026 13:01:14 -0800 Subject: [PATCH 12/54] [Fix] Enhance MidStreamFallbackError to preserve original status code and attributes - Updated MidStreamFallbackError to retrieve and maintain the original status code from the wrapped exception. - Ensured that message, request, and response fields remain consistent after calling the parent constructor. - Added unit tests to verify the correct propagation of status codes and attributes in various scenarios. --- litellm/exceptions.py | 20 ++++++++++-- tests/local_testing/test_exceptions.py | 42 ++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index eb027334606..edbb8b88915 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -955,7 +955,8 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore generated_content: str = "", is_pre_first_chunk: bool = False, ): - self.status_code = 503 # Service Unavailable + original_status = getattr(original_exception, "status_code", None) + self.status_code = int(original_status) if original_status is not None else 503 self.message = f"litellm.MidStreamFallbackError: {message}" self.model = model self.llm_provider = llm_provider @@ -978,7 +979,14 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore else: self.response = response - # Call the parent constructor + # Save the original attributes before they are overridden by ServiceUnavailableError + _saved_response = self.response + _saved_request = getattr(self.response, "request", None) or httpx.Request( + method="POST", url=f"https://{llm_provider}.com/v1/" + ) + _saved_message = self.message + + # Call the parent constructor (which hardcodes status_code=503 and modifies the response object) super().__init__( message=self.message, llm_provider=llm_provider, @@ -988,6 +996,14 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore max_retries=self.max_retries, num_retries=self.num_retries, ) + + # Restore the propagated status and original response/request objects + self.status_code = int(original_status) if original_status is not None else 503 + self.response = _saved_response + self.request = _saved_request + self.message = _saved_message + self.args = (_saved_message,) + self.args = (_saved_message,) def __str__(self): _message = self.message diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index 4cc2723ace8..567b2d10480 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -1422,4 +1422,46 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model): assert exc_info.value.type == "invalid_request_error" +def test_midstream_fallback_error_status_code_propagation(): + """ + MidStreamFallbackError should preserve the original status code and keep + message/request/response fields consistent after super().__init__(). + """ + import litellm + from httpx import Request, Response + # 1) Wrapping a 429 should preserve the 429 status + original_req = Request("POST", "https://api.openai.com/v1/chat/completions") + original_resp = Response(status_code=429, request=original_req) + + rate_limit_error = litellm.RateLimitError( + message="Rate limit exceeded", + llm_provider="openai", + model="gpt-4o-mini", + response=original_resp, + ) + + midstream_error = litellm.exceptions.MidStreamFallbackError( + message="stream broke", + model="gpt-4o-mini", + llm_provider="openai", + original_exception=rate_limit_error, + ) + + assert midstream_error.status_code == 429 + assert midstream_error.response.status_code == 429 + assert str(midstream_error.response.request.url) == "https://openai.com/v1/" + assert midstream_error.message == "litellm.MidStreamFallbackError: stream broke" + assert midstream_error.args == ("litellm.MidStreamFallbackError: stream broke",) + + # 2) With no original exception, should default to 503 + midstream_fallback = litellm.exceptions.MidStreamFallbackError( + message="stream broke without original", + model="gpt-4o-mini", + llm_provider="openai", + original_exception=None, + ) + + assert midstream_fallback.status_code == 503 + assert midstream_fallback.response.status_code == 503 + assert str(midstream_fallback.response.request.url) == "https://openai.com/v1/" \ No newline at end of file From 0e014253d7670df86cba872ffe0b87930a45b0a7 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Thu, 26 Feb 2026 15:07:51 -0600 Subject: [PATCH 13/54] refactor(cost): dedupe image token usage cost helper - extract shared calculate_image_response_cost_from_usage() helper\n- reuse helper in vertex and gemini image generation cost calculators\n- preserve provider-specific fallback to output_cost_per_image --- .../litellm_core_utils/llm_cost_calc/utils.py | 60 +++++++++++++++ .../image_generation/cost_calculator.py | 74 +++---------------- .../image_generation/cost_calculator.py | 72 ++---------------- 3 files changed, 76 insertions(+), 130 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index a9fd0f4ea8a..bf0b2709365 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -8,9 +8,11 @@ from litellm._logging import verbose_logger from litellm.types.utils import ( CacheCreationTokenDetails, CallTypes, + CompletionTokensDetailsWrapper, ImageResponse, ModelInfo, PassthroughCallTypes, + PromptTokensDetailsWrapper, ServiceTier, Usage, ) @@ -767,6 +769,64 @@ def generic_cost_per_token( # noqa: PLR0915 return prompt_cost, completion_cost +def calculate_image_response_cost_from_usage( + model: str, + image_response: ImageResponse, + custom_llm_provider: str, +) -> Optional[float]: + """ + Calculate image generation cost from usage metadata when available. + + Returns: + Optional[float]: total cost from token usage, or None when usage metadata + is missing/incomplete and caller should fall back to flat per-image pricing. + """ + usage = image_response.usage + if usage is None: + return None + + prompt_tokens = usage.input_tokens + completion_tokens = usage.output_tokens + total_tokens = usage.total_tokens + + if prompt_tokens is None or completion_tokens is None or total_tokens is None: + return None + + # ImageResponse may carry a default zeroed usage object even when provider + # usage metadata is absent. Treat this as missing usage and fall back. + if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: + return None + + input_tokens_details = getattr(usage, "input_tokens_details", None) + prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None + if input_tokens_details is not None: + prompt_tokens_details = PromptTokensDetailsWrapper( + text_tokens=getattr(input_tokens_details, "text_tokens", None), + image_tokens=getattr(input_tokens_details, "image_tokens", None), + cached_tokens=0, + ) + + normalized_usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + prompt_tokens_details=prompt_tokens_details, + completion_tokens_details=CompletionTokensDetailsWrapper( + text_tokens=0, + image_tokens=completion_tokens, + reasoning_tokens=0, + audio_tokens=0, + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=normalized_usage, + custom_llm_provider=custom_llm_provider, + ) + return prompt_cost + completion_cost + + class CostCalculatorUtils: @staticmethod def _call_type_has_image_response(call_type: str) -> bool: diff --git a/litellm/llms/gemini/image_generation/cost_calculator.py b/litellm/llms/gemini/image_generation/cost_calculator.py index 6d7572d5522..941ab0d50f7 100644 --- a/litellm/llms/gemini/image_generation/cost_calculator.py +++ b/litellm/llms/gemini/image_generation/cost_calculator.py @@ -2,71 +2,13 @@ Google AI Image Generation Cost Calculator """ -from typing import Any, Optional +from typing import Any import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token -from litellm.types.utils import ( - CompletionTokensDetailsWrapper, - ImageResponse, - PromptTokensDetailsWrapper, - Usage, +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + calculate_image_response_cost_from_usage, ) - - -def _calculate_token_based_cost(model: str, image_response: ImageResponse) -> Optional[float]: - """ - Calculate token-based image generation cost when usage metadata is available. - - Falls back to None when usage metadata is missing/incomplete. - """ - usage = image_response.usage - if usage is None: - return None - - prompt_tokens = usage.input_tokens - completion_tokens = usage.output_tokens - total_tokens = usage.total_tokens - - if ( - prompt_tokens is None - or completion_tokens is None - or total_tokens is None - ): - return None - # ImageResponse may carry a default zeroed usage object even when provider - # usage metadata is absent. Treat this as missing usage and fall back. - if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: - return None - - input_tokens_details = getattr(usage, "input_tokens_details", None) - prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None - if input_tokens_details is not None: - prompt_tokens_details = PromptTokensDetailsWrapper( - text_tokens=getattr(input_tokens_details, "text_tokens", None), - image_tokens=getattr(input_tokens_details, "image_tokens", None), - cached_tokens=0, - ) - - normalized_usage = Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - prompt_tokens_details=prompt_tokens_details, - completion_tokens_details=CompletionTokensDetailsWrapper( - text_tokens=0, - image_tokens=completion_tokens, - reasoning_tokens=0, - audio_tokens=0, - ), - ) - - prompt_cost, completion_cost = generic_cost_per_token( - model=model, - usage=normalized_usage, - custom_llm_provider="gemini", - ) - return prompt_cost + completion_cost +from litellm.types.utils import ImageResponse def cost_calculator( @@ -74,7 +16,7 @@ def cost_calculator( image_response: Any, ) -> float: """ - Vertex AI Image Generation Cost Calculator + Google AI Image Generation Cost Calculator """ _model_info = litellm.get_model_info( model=model, @@ -82,8 +24,10 @@ def cost_calculator( ) if isinstance(image_response, ImageResponse): - token_based_cost = _calculate_token_based_cost( - model=model, image_response=image_response + token_based_cost = calculate_image_response_cost_from_usage( + model=model, + image_response=image_response, + custom_llm_provider="gemini", ) if token_based_cost is not None: return token_based_cost diff --git a/litellm/llms/vertex_ai/image_generation/cost_calculator.py b/litellm/llms/vertex_ai/image_generation/cost_calculator.py index ac587182f04..012de5498cb 100644 --- a/litellm/llms/vertex_ai/image_generation/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_generation/cost_calculator.py @@ -2,71 +2,11 @@ Vertex AI Image Generation Cost Calculator """ -from typing import Optional - import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token -from litellm.types.utils import ( - CompletionTokensDetailsWrapper, - ImageResponse, - PromptTokensDetailsWrapper, - Usage, +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + calculate_image_response_cost_from_usage, ) - - -def _calculate_token_based_cost(model: str, image_response: ImageResponse) -> Optional[float]: - """ - Calculate token-based image generation cost when usage metadata is available. - - Falls back to None when usage metadata is missing/incomplete. - """ - usage = image_response.usage - if usage is None: - return None - - prompt_tokens = usage.input_tokens - completion_tokens = usage.output_tokens - total_tokens = usage.total_tokens - - if ( - prompt_tokens is None - or completion_tokens is None - or total_tokens is None - ): - return None - # ImageResponse may carry a default zeroed usage object even when provider - # usage metadata is absent. Treat this as missing usage and fall back. - if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: - return None - - input_tokens_details = getattr(usage, "input_tokens_details", None) - prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None - if input_tokens_details is not None: - prompt_tokens_details = PromptTokensDetailsWrapper( - text_tokens=getattr(input_tokens_details, "text_tokens", None), - image_tokens=getattr(input_tokens_details, "image_tokens", None), - cached_tokens=0, - ) - - normalized_usage = Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - prompt_tokens_details=prompt_tokens_details, - completion_tokens_details=CompletionTokensDetailsWrapper( - text_tokens=0, - image_tokens=completion_tokens, - reasoning_tokens=0, - audio_tokens=0, - ), - ) - - prompt_cost, completion_cost = generic_cost_per_token( - model=model, - usage=normalized_usage, - custom_llm_provider="vertex_ai", - ) - return prompt_cost + completion_cost +from litellm.types.utils import ImageResponse def cost_calculator( @@ -81,8 +21,10 @@ def cost_calculator( custom_llm_provider="vertex_ai", ) - token_based_cost = _calculate_token_based_cost( - model=model, image_response=image_response + token_based_cost = calculate_image_response_cost_from_usage( + model=model, + image_response=image_response, + custom_llm_provider="vertex_ai", ) if token_based_cost is not None: return token_based_cost From c651a511bd5432a6b92afac3d6cba1944daca809 Mon Sep 17 00:00:00 2001 From: Dharamendra Kumar Date: Thu, 26 Feb 2026 13:26:51 -0800 Subject: [PATCH 14/54] Update test to righ place --- .../test_exception_header_preservation.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/test_litellm/test_exception_header_preservation.py b/tests/test_litellm/test_exception_header_preservation.py index d3e33fa13b3..ec52d9fb746 100644 --- a/tests/test_litellm/test_exception_header_preservation.py +++ b/tests/test_litellm/test_exception_header_preservation.py @@ -16,6 +16,8 @@ from litellm.exceptions import ( ContentPolicyViolationError, ContextWindowExceededError, ImageFetchError, + MidStreamFallbackError, + RateLimitError, ) @@ -210,6 +212,46 @@ class TestExceptionAttributes: assert error.num_retries == 1 assert error.status_code == 400 + def test_midstream_fallback_error_status_code_propagation(self): + """ + MidStreamFallbackError should preserve the original status code and keep + message/request/response fields consistent after super().__init__(). + """ + original_req = httpx.Request("POST", "https://api.openai.com/v1/chat/completions") + original_resp = httpx.Response(status_code=429, request=original_req) + + rate_limit_error = RateLimitError( + message="Rate limit exceeded", + llm_provider="openai", + model="gpt-4o-mini", + response=original_resp, + ) + + midstream_error = MidStreamFallbackError( + message="stream broke", + model="gpt-4o-mini", + llm_provider="openai", + original_exception=rate_limit_error, + ) + + assert midstream_error.status_code == 429 + assert midstream_error.response.status_code == 429 + assert str(midstream_error.response.request.url) == "https://openai.com/v1/" + assert midstream_error.message == "litellm.MidStreamFallbackError: stream broke" + assert midstream_error.args == ("litellm.MidStreamFallbackError: stream broke",) + + # With no original exception, should default to 503. + midstream_fallback = MidStreamFallbackError( + message="stream broke without original", + model="gpt-4o-mini", + llm_provider="openai", + original_exception=None, + ) + + assert midstream_fallback.status_code == 503 + assert midstream_fallback.response.status_code == 503 + assert str(midstream_fallback.response.request.url) == "https://openai.com/v1/" + class TestProxyHeaderExtraction: """Test that proxy correctly extracts headers from exceptions.""" From 840333f32e4006a23029d5df86a10e01d093d98a Mon Sep 17 00:00:00 2001 From: Dharamendra Kumar Date: Thu, 26 Feb 2026 13:31:37 -0800 Subject: [PATCH 15/54] Restore --- tests/local_testing/test_exceptions.py | 42 -------------------------- 1 file changed, 42 deletions(-) diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index 567b2d10480..4cc2723ace8 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -1422,46 +1422,4 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model): assert exc_info.value.type == "invalid_request_error" -def test_midstream_fallback_error_status_code_propagation(): - """ - MidStreamFallbackError should preserve the original status code and keep - message/request/response fields consistent after super().__init__(). - """ - import litellm - from httpx import Request, Response - # 1) Wrapping a 429 should preserve the 429 status - original_req = Request("POST", "https://api.openai.com/v1/chat/completions") - original_resp = Response(status_code=429, request=original_req) - - rate_limit_error = litellm.RateLimitError( - message="Rate limit exceeded", - llm_provider="openai", - model="gpt-4o-mini", - response=original_resp, - ) - - midstream_error = litellm.exceptions.MidStreamFallbackError( - message="stream broke", - model="gpt-4o-mini", - llm_provider="openai", - original_exception=rate_limit_error, - ) - - assert midstream_error.status_code == 429 - assert midstream_error.response.status_code == 429 - assert str(midstream_error.response.request.url) == "https://openai.com/v1/" - assert midstream_error.message == "litellm.MidStreamFallbackError: stream broke" - assert midstream_error.args == ("litellm.MidStreamFallbackError: stream broke",) - - # 2) With no original exception, should default to 503 - midstream_fallback = litellm.exceptions.MidStreamFallbackError( - message="stream broke without original", - model="gpt-4o-mini", - llm_provider="openai", - original_exception=None, - ) - - assert midstream_fallback.status_code == 503 - assert midstream_fallback.response.status_code == 503 - assert str(midstream_fallback.response.request.url) == "https://openai.com/v1/" \ No newline at end of file From fcdfc638b01ec2dcb6324e2698fdda02b633c3ad Mon Sep 17 00:00:00 2001 From: Dharamendra Kumar Date: Thu, 26 Feb 2026 13:44:06 -0800 Subject: [PATCH 16/54] Remove nit --- litellm/exceptions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index edbb8b88915..b36d4ef877c 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1003,7 +1003,6 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore self.request = _saved_request self.message = _saved_message self.args = (_saved_message,) - self.args = (_saved_message,) def __str__(self): _message = self.message From f4e3e016a1c534996e8d901e67b9e547eab0a892 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 16:30:41 -0800 Subject: [PATCH 17/54] fixing inf budget --- litellm/integrations/prometheus.py | 2 + .../test_prometheus_user_team_metrics.py | 137 +++++++++++++++++- 2 files changed, 138 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 08db77e8571..121431d2114 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2686,6 +2686,8 @@ class PrometheusLogger(CustomLogger): if team_info: team_object.budget_reset_at = team_info.budget_reset_at + if team_object.max_budget is None and team_info.max_budget is not None: + team_object.max_budget = team_info.max_budget return team_object diff --git a/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py b/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py index a840e2fe162..40e33995c66 100644 --- a/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py +++ b/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py @@ -1,7 +1,8 @@ """ Unit tests for Prometheus user and team count metrics """ -from unittest.mock import MagicMock +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch import pytest from prometheus_client import REGISTRY @@ -258,3 +259,137 @@ class TestPrometheusUserTeamCountMetrics: assert True except Exception as e: pytest.fail(f"Metrics should handle large values: {e}") + + +# --------------------------------------------------------------------------- +# Regression tests: team budget showing +Inf when user_api_key_team_max_budget +# is None in request metadata but the team has a real budget in the DB. +# --------------------------------------------------------------------------- + + +async def test_assemble_team_object_uses_db_max_budget_when_metadata_is_none( + prometheus_logger, +): + """ + When max_budget is None in request metadata (e.g. stale key cache), + _assemble_team_object must fall back to the value returned by get_team_object + so that _safe_get_remaining_budget does not return +Inf. + """ + db_team = MagicMock() + db_team.max_budget = 3000.0 + db_team.budget_reset_at = datetime(2026, 3, 1, tzinfo=timezone.utc) + + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + mock_get_team.return_value = db_team + team_object = await prometheus_logger._assemble_team_object( + team_id="c5c33858-4379-4c90-8733-d9c58c312c10", + team_alias="ai-ml-local_dev", + spend=1617.02, + max_budget=None, # simulates None coming from request metadata + response_cost=0.5, + ) + + assert team_object.max_budget == 3000.0, ( + "max_budget should be populated from DB when metadata value is None" + ) + assert team_object.budget_reset_at == datetime(2026, 3, 1, tzinfo=timezone.utc) + + +async def test_assemble_team_object_does_not_override_metadata_max_budget( + prometheus_logger, +): + """ + When max_budget IS present in request metadata, it must not be overridden + by the DB value. + """ + db_team = MagicMock() + db_team.max_budget = 9999.0 + db_team.budget_reset_at = None + + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + mock_get_team.return_value = db_team + team_object = await prometheus_logger._assemble_team_object( + team_id="team-1", + team_alias="my-team", + spend=50.0, + max_budget=100.0, # metadata has a real value + response_cost=1.0, + ) + + assert team_object.max_budget == 100.0, ( + "max_budget from metadata must not be replaced by the DB value" + ) + + +async def test_set_team_budget_metrics_after_api_request_no_inf_when_metadata_budget_none( + prometheus_logger, +): + """ + End-to-end: when user_api_key_team_max_budget is None in request metadata + but the team has a real budget in the DB, the metric must NOT be set to +Inf. + """ + prometheus_logger.litellm_remaining_team_budget_metric = MagicMock() + prometheus_logger.litellm_team_max_budget_metric = MagicMock() + prometheus_logger.litellm_team_budget_remaining_hours_metric = MagicMock() + + db_team = MagicMock() + db_team.max_budget = 3000.0 + db_team.budget_reset_at = datetime(2026, 3, 1, tzinfo=timezone.utc) + + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + mock_get_team.return_value = db_team + await prometheus_logger._set_team_budget_metrics_after_api_request( + user_api_team="c5c33858-4379-4c90-8733-d9c58c312c10", + user_api_team_alias="ai-ml-local_dev", + team_spend=1617.02, + team_max_budget=None, # simulates stale key cache + response_cost=0.5, + ) + + set_call_args = ( + prometheus_logger.litellm_remaining_team_budget_metric.labels().set.call_args + ) + assert set_call_args is not None, "remaining_team_budget_metric.labels().set was not called" + actual_value = set_call_args[0][0] + assert actual_value != float("inf"), ( + f"remaining_team_budget_metric must not be +Inf when team has a real budget; got {actual_value}" + ) + expected = 3000.0 - 1617.02 - 0.5 + assert abs(actual_value - expected) < 0.01, ( + f"Expected remaining budget ~{expected}, got {actual_value}" + ) + + +async def test_set_team_budget_metrics_after_api_request_inf_when_genuinely_no_budget( + prometheus_logger, +): + """ + When the team genuinely has no budget (max_budget=None in both metadata and + DB), +Inf is the correct value and must be preserved. + """ + prometheus_logger.litellm_remaining_team_budget_metric = MagicMock() + prometheus_logger.litellm_team_max_budget_metric = MagicMock() + prometheus_logger.litellm_team_budget_remaining_hours_metric = MagicMock() + + db_team = MagicMock() + db_team.max_budget = None + db_team.budget_reset_at = None + + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + mock_get_team.return_value = db_team + await prometheus_logger._set_team_budget_metrics_after_api_request( + user_api_team="team-no-budget", + user_api_team_alias="no-budget-team", + team_spend=10.0, + team_max_budget=None, + response_cost=1.0, + ) + + set_call_args = ( + prometheus_logger.litellm_remaining_team_budget_metric.labels().set.call_args + ) + assert set_call_args is not None + actual_value = set_call_args[0][0] + assert actual_value == float("inf"), ( + "remaining_team_budget_metric should be +Inf when team truly has no budget" + ) From 0e1428b59d496a1f375cdb3a136f7e8e9a1b0127 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 16:51:18 -0800 Subject: [PATCH 18/54] remove orphan comment from test file Co-Authored-By: Claude Haiku 4.5 --- .../enterprise_callbacks/test_prometheus_logging_callbacks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index 08b9351f9a3..61b1b1f8185 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -2316,5 +2316,3 @@ async def test_prometheus_token_metrics_with_prometheus_config(): raise AssertionError(f"Metric {metric_name} not found in registry") print("✓ All token metrics validated successfully!") - - # check final value of metrics in registry From 28c77b48c9a6a3342fb662ccb0d705c61b3c9e69 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 16:57:31 -0800 Subject: [PATCH 19/54] fix +Inf user budget metric when metadata max_budget is None Same bug as team budget: _assemble_user_object fetched user info from DB but only used budget_reset_at, discarding max_budget. When the key cache has a stale None for user_max_budget, _safe_get_remaining_budget returns +Inf. Now falls back to DB max_budget when metadata value is None. Co-Authored-By: Claude Sonnet 4.6 --- litellm/integrations/prometheus.py | 2 + .../test_prometheus_user_team_metrics.py | 130 ++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 121431d2114..7a08432b9a1 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2905,6 +2905,8 @@ class PrometheusLogger(CustomLogger): if user_info: user_object.budget_reset_at = user_info.budget_reset_at + if user_object.max_budget is None and user_info.max_budget is not None: + user_object.max_budget = user_info.max_budget return user_object diff --git a/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py b/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py index 40e33995c66..cd76ba1e863 100644 --- a/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py +++ b/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py @@ -393,3 +393,133 @@ async def test_set_team_budget_metrics_after_api_request_inf_when_genuinely_no_b assert actual_value == float("inf"), ( "remaining_team_budget_metric should be +Inf when team truly has no budget" ) + + +# --------------------------------------------------------------------------- +# Regression tests: user budget showing +Inf when user_api_key_user_max_budget +# is None in request metadata but the user has a real budget in the DB. +# --------------------------------------------------------------------------- + + +async def test_assemble_user_object_uses_db_max_budget_when_metadata_is_none( + prometheus_logger, +): + """ + When max_budget is None in request metadata (e.g. stale key cache), + _assemble_user_object must fall back to the value returned by get_user_object + so that _safe_get_remaining_budget does not return +Inf. + """ + db_user = MagicMock() + db_user.max_budget = 500.0 + db_user.budget_reset_at = datetime(2026, 3, 1, tzinfo=timezone.utc) + + with patch("litellm.proxy.auth.auth_checks.get_user_object") as mock_get_user: + mock_get_user.return_value = db_user + user_object = await prometheus_logger._assemble_user_object( + user_id="user-abc-123", + spend=120.0, + max_budget=None, # simulates None coming from request metadata + response_cost=0.5, + ) + + assert user_object.max_budget == 500.0, ( + "max_budget should be populated from DB when metadata value is None" + ) + assert user_object.budget_reset_at == datetime(2026, 3, 1, tzinfo=timezone.utc) + + +async def test_assemble_user_object_does_not_override_metadata_max_budget( + prometheus_logger, +): + """ + When max_budget IS present in request metadata, it must not be overridden + by the DB value. + """ + db_user = MagicMock() + db_user.max_budget = 9999.0 + db_user.budget_reset_at = None + + with patch("litellm.proxy.auth.auth_checks.get_user_object") as mock_get_user: + mock_get_user.return_value = db_user + user_object = await prometheus_logger._assemble_user_object( + user_id="user-abc-123", + spend=50.0, + max_budget=100.0, # metadata has a real value + response_cost=1.0, + ) + + assert user_object.max_budget == 100.0, ( + "max_budget from metadata must not be replaced by the DB value" + ) + + +async def test_set_user_budget_metrics_after_api_request_no_inf_when_metadata_budget_none( + prometheus_logger, +): + """ + End-to-end: when user_max_budget is None in request metadata but the user + has a real budget in the DB, the metric must NOT be set to +Inf. + """ + prometheus_logger.litellm_remaining_user_budget_metric = MagicMock() + prometheus_logger.litellm_user_max_budget_metric = MagicMock() + prometheus_logger.litellm_user_budget_remaining_hours_metric = MagicMock() + + db_user = MagicMock() + db_user.max_budget = 500.0 + db_user.budget_reset_at = datetime(2026, 3, 1, tzinfo=timezone.utc) + + with patch("litellm.proxy.auth.auth_checks.get_user_object") as mock_get_user: + mock_get_user.return_value = db_user + await prometheus_logger._set_user_budget_metrics_after_api_request( + user_id="user-abc-123", + user_spend=120.0, + user_max_budget=None, # simulates stale key cache + response_cost=0.5, + ) + + set_call_args = ( + prometheus_logger.litellm_remaining_user_budget_metric.labels().set.call_args + ) + assert set_call_args is not None, "remaining_user_budget_metric.labels().set was not called" + actual_value = set_call_args[0][0] + assert actual_value != float("inf"), ( + f"remaining_user_budget_metric must not be +Inf when user has a real budget; got {actual_value}" + ) + expected = 500.0 - 120.0 - 0.5 + assert abs(actual_value - expected) < 0.01, ( + f"Expected remaining budget ~{expected}, got {actual_value}" + ) + + +async def test_set_user_budget_metrics_after_api_request_inf_when_genuinely_no_budget( + prometheus_logger, +): + """ + When the user genuinely has no budget (max_budget=None in both metadata and + DB), +Inf is the correct value and must be preserved. + """ + prometheus_logger.litellm_remaining_user_budget_metric = MagicMock() + prometheus_logger.litellm_user_max_budget_metric = MagicMock() + prometheus_logger.litellm_user_budget_remaining_hours_metric = MagicMock() + + db_user = MagicMock() + db_user.max_budget = None + db_user.budget_reset_at = None + + with patch("litellm.proxy.auth.auth_checks.get_user_object") as mock_get_user: + mock_get_user.return_value = db_user + await prometheus_logger._set_user_budget_metrics_after_api_request( + user_id="user-no-budget", + user_spend=10.0, + user_max_budget=None, + response_cost=1.0, + ) + + set_call_args = ( + prometheus_logger.litellm_remaining_user_budget_metric.labels().set.call_args + ) + assert set_call_args is not None + actual_value = set_call_args[0][0] + assert actual_value == float("inf"), ( + "remaining_user_budget_metric should be +Inf when user truly has no budget" + ) From 48b9ecacadf257c23605efdcd283465bf13912d1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 26 Feb 2026 17:06:07 -0800 Subject: [PATCH 20/54] fix(realtime): fix guardrails not firing for Gemini/Vertex AI and provider_config realtime WebSocket sessions (#22168) * fix(gemini): enable inputAudioTranscription and handle transcription events for realtime guardrails Gemini sends inputTranscription/outputTranscription inside serverContent separately from modelTurn/turnComplete. This adds handling to convert them into OpenAI-compatible events so the guardrail pipeline can inspect voice input, and enables inputAudioTranscription in the session setup config. Made-with: Cursor * fix(vertex_ai): enable inputAudioTranscription in realtime session config Add inputAudioTranscription to the Vertex AI realtime setup so the backend returns transcripts of user speech, allowing guardrails to inspect voice input. Made-with: Cursor * fix(realtime): pass user_api_key_dict and guardrail metadata through async_realtime handler The base LLM HTTP handler's async_realtime method was not accepting or forwarding user_api_key_dict and litellm_metadata to RealTimeStreaming. This meant guardrails configured with default_on=false were silently skipped for all provider_config-based realtime connections (Gemini, Vertex AI, etc). Also fixes wss:// connections when SSL_VERIFY=False by overriding ssl=False for secure WebSocket URLs. Made-with: Cursor * fix(realtime): forward guardrail metadata for generic provider_config and vertex_ai paths The _arealtime function was not passing user_api_key_dict or litellm_metadata to base_llm_http_handler.async_realtime() for the generic provider_config path and the vertex_ai-specific path. This broke guardrail resolution since RealTimeStreaming.request_data was empty, causing should_run_guardrail to return False. Made-with: Cursor * fix(realtime): voice guardrail responses and block duplicate response.create on text input When a guardrail blocks voice input, send a conversation.item.create + response.create to the backend so the LLM voices the guardrail message as audio instead of only returning text. Also adds pending_guardrail_message tracking to suppress the automatic response.create the client sends after a blocked text message, and broadens _has_audio_transcription_guardrails to match pre_call/post_call modes. Made-with: Cursor * test(realtime): update guardrail tests for broadened audio transcription check and add integration tests Update existing tests to reflect that pre_call guardrails now correctly trigger the audio/VAD session.update injection. Add integration test file for live OpenAI realtime guardrail testing. Made-with: Cursor * fix(realtime): instruct LLM to say exact guardrail message verbatim The previous prompt gave the LLM creative freedom to paraphrase the guardrail violation message. Now it instructs the LLM to repeat the exact configured message word for word. Made-with: Cursor * fix(realtime): preserve wss ssl semantics and move live guardrail test Keep TLS enabled for wss realtime sessions while honoring SSL_VERIFY=False via a no-verify SSLContext, move the OpenAI live guardrail test into llm_translation, and dedupe duplicated guardrail-detection helpers to prevent drift. Made-with: Cursor --- .../litellm_core_utils/realtime_streaming.py | 70 ++-- litellm/llms/custom_httpx/llm_http_handler.py | 13 + .../llms/gemini/realtime/transformation.py | 48 +++ .../llms/vertex_ai/realtime/transformation.py | 2 + litellm/realtime_api/main.py | 4 + .../test_realtime_guardrails_openai.py | 355 ++++++++++++++++++ .../test_realtime_streaming.py | 21 +- 7 files changed, 481 insertions(+), 32 deletions(-) create mode 100644 tests/llm_translation/realtime/test_realtime_guardrails_openai.py diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 449a4892621..294f9c485c1 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -72,6 +72,9 @@ class RealTimeStreaming: self.request_data: Dict = request_data or {} # Violation counter for end_session_after_n_fails support self._violation_count: int = 0 + # When a text message is blocked, hold the guardrail reason so the next + # response.create can be rewritten to include the failure context. + self._pending_guardrail_message: Optional[str] = None def _should_store_message( self, @@ -261,18 +264,12 @@ class RealTimeStreaming: When this returns True, we inject a session.update to disable the LLM's auto-response so the guardrail can gate it first. - """ - from litellm.integrations.custom_guardrail import CustomGuardrail - from litellm.types.guardrails import GuardrailEventHooks - return any( - isinstance(cb, CustomGuardrail) - and cb.should_run_guardrail( - data=self.request_data, - event_type=GuardrailEventHooks.realtime_input_transcription, - ) - for cb in litellm.callbacks - ) + Must match the same hook criteria as run_realtime_guardrails() so that + any guardrail that would actually check the transcript also disables + auto-response before the transcript arrives. + """ + return self._has_realtime_guardrails() async def run_realtime_guardrails( self, @@ -335,18 +332,35 @@ class RealTimeStreaming: # Use realtime_violation_message if configured; fall back to guardrail error text. error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg - # Return the error directly to the WebSocket consumer. + # Cancel any in-progress LLM response (e.g. VAD auto-response). + await self._send_to_backend(json.dumps({"type": "response.cancel"})) + # Send the policy violation hint (shows as small gray status text in UI). await self.websocket.send_text( - json.dumps( - { - "type": "error", - "error": { - "type": "guardrail_violation", - "message": error_msg, - "code": "content_policy_violation", - }, - } - ) + json.dumps({ + "type": "error", + "error": { + "type": "guardrail_violation", + "message": error_msg, + "code": "content_policy_violation", + }, + }) + ) + # Ask the LLM to voice the exact guardrail message so the + # user hears it as audio in voice sessions (not just text). + guardrail_prompt = ( + f"Say exactly the following message to the user, word for word, " + f"do not add anything else: {error_msg}" + ) + await self._send_to_backend(json.dumps({ + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": guardrail_prompt}], + }, + })) + await self._send_to_backend( + json.dumps({"type": "response.create"}) ) self._violation_count += 1 @@ -559,7 +573,17 @@ class RealTimeStreaming: combined_text ) if blocked: - continue # don't forward to backend + # Store the guardrail reason so the next response.create + # (sent automatically by the client) is rewritten to + # include it as response instructions. + self._pending_guardrail_message = combined_text + continue # don't forward the original blocked message + + if msg_type == "response.create" and self._pending_guardrail_message: + # The guardrail already sent the synthetic AI bubble — drop this + # response.create so OpenAI doesn't generate an additional response. + self._pending_guardrail_message = None + continue except (json.JSONDecodeError, AttributeError): pass diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index b09a36be60f..d6fdc58099f 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1,4 +1,5 @@ import json +import ssl from typing import ( TYPE_CHECKING, Any, @@ -4659,6 +4660,8 @@ class BaseLLMHTTPHandler: api_key: Optional[str] = None, client: Optional[Any] = None, timeout: Optional[float] = None, + user_api_key_dict: Optional[Any] = None, + litellm_metadata: Optional[Dict[str, Any]] = None, ): import websockets from websockets.asyncio.client import ClientConnection @@ -4672,6 +4675,11 @@ class BaseLLMHTTPHandler: try: ssl_context = get_shared_realtime_ssl_context() + if url.startswith("wss://") and ssl_context is False: + # Keep TLS for wss:// while honoring SSL_VERIFY=False semantics. + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE async with websockets.connect( # type: ignore url, additional_headers=headers, @@ -4686,12 +4694,17 @@ class BaseLLMHTTPHandler: if _session_config: await backend_ws.send(_session_config) + _request_data: Dict[str, Any] = {} + if litellm_metadata: + _request_data["litellm_metadata"] = litellm_metadata realtime_streaming = RealTimeStreaming( websocket, cast(ClientConnection, backend_ws), logging_obj, provider_config, model, + user_api_key_dict=user_api_key_dict, + request_data=_request_data, ) if _session_config: realtime_streaming.session_configuration_request = _session_config diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 2e0e678e69f..d9465c95e3b 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -867,6 +867,52 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) returned_message: List[OpenAIRealtimeEvents] = [] + # Handle transcription events that arrive independently from model + # content. Gemini sends inputTranscription / outputTranscription + # inside serverContent, separately from modelTurn / turnComplete. + server_content = json_message.get("serverContent") + if isinstance(server_content, dict): + input_tx = server_content.get("inputTranscription") + if isinstance(input_tx, dict) and input_tx.get("text"): + returned_message.append( + cast(OpenAIRealtimeEvents, { + "type": "conversation.item.input_audio_transcription.completed", + "event_id": "event_{}".format(uuid.uuid4()), + "transcript": input_tx["text"], + "item_id": "item_{}".format(uuid.uuid4()), + "content_index": 0, + }) + ) + + output_tx = server_content.get("outputTranscription") + if isinstance(output_tx, dict) and output_tx.get("text"): + returned_message.append( + cast(OpenAIRealtimeEvents, { + "type": "response.audio_transcript.delta", + "event_id": "event_{}".format(uuid.uuid4()), + "delta": output_tx["text"], + "item_id": current_output_item_id or "item_{}".format(uuid.uuid4()), + "response_id": current_response_id or "resp_{}".format(uuid.uuid4()), + "output_index": 0, + "content_index": 0, + }) + ) + + # If serverContent only contained transcription(s) and no model + # content, return early — the main loop would fail on unknown keys. + _model_content_keys = {"modelTurn", "turnComplete", "interrupted", "generationComplete"} + if not any(k in server_content for k in _model_content_keys): + return { + "response": returned_message, + "current_output_item_id": current_output_item_id, + "current_response_id": current_response_id, + "current_delta_chunks": current_delta_chunks, + "current_conversation_id": current_conversation_id, + "current_item_chunks": current_item_chunks, + "current_delta_type": current_delta_type, + "session_configuration_request": session_configuration_request, + } + for key, value in json_message.items(): # Check if this key or any nested key matches our mapping openai_event = self.map_openai_event( @@ -974,6 +1020,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): setup_config: BidiGenerateContentSetup = { "model": f"models/{model}", "generationConfig": {"responseModalities": response_modalities}, + # Return input transcript so guardrails can inspect user speech. + "inputAudioTranscription": {}, } if output_audio_transcription: setup_config["outputAudioTranscription"] = {} diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index eaa9844f108..5eae143175b 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -124,6 +124,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): "silenceDurationMs": 800, } }, + # Return input transcript so guardrails can inspect user speech. + "inputAudioTranscription": {}, # Return output transcript so clients can read what the model said. "outputAudioTranscription": {}, } diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index ac597fc623d..3e64f61abdb 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -106,6 +106,8 @@ async def _arealtime( # noqa: PLR0915 client=client, timeout=timeout, headers=headers, + user_api_key_dict=kwargs.get("user_api_key_dict"), + litellm_metadata=_build_litellm_metadata(kwargs), ) elif _custom_llm_provider == "azure": api_base = ( @@ -277,6 +279,8 @@ async def _arealtime( # noqa: PLR0915 client=client, timeout=timeout, headers=headers, + user_api_key_dict=kwargs.get("user_api_key_dict"), + litellm_metadata=_build_litellm_metadata(kwargs), ) else: raise ValueError(f"Unsupported model: {model}") diff --git a/tests/llm_translation/realtime/test_realtime_guardrails_openai.py b/tests/llm_translation/realtime/test_realtime_guardrails_openai.py new file mode 100644 index 00000000000..a7913e6d761 --- /dev/null +++ b/tests/llm_translation/realtime/test_realtime_guardrails_openai.py @@ -0,0 +1,355 @@ +""" +Integration tests for RealTimeStreaming guardrails against a live OpenAI backend. + +These tests require OPENAI_API_KEY and are skipped if not set. + +They verify end-to-end that: + 1. A text message blocked by a guardrail -> error event sent to client, NO AI response. + 2. A voice transcript blocked by a guardrail -> error event sent, response.create NOT sent. + 3. A clean text message passes through and triggers a real OpenAI response. + +Run with: + poetry run pytest tests/llm_translation/realtime/test_realtime_guardrails_openai.py -v -s +""" + +import asyncio +import json +import os +from typing import List +from unittest.mock import AsyncMock, MagicMock + +import pytest + +import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming +from litellm.types.guardrails import GuardrailEventHooks + +OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY") +OPENAI_REALTIME_URL = ( + "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-12-17" +) + +pytestmark = pytest.mark.skipif( + not OPENAI_API_KEY, + reason="OPENAI_API_KEY not set - skipping OpenAI realtime integration tests", +) + +# A unique phrase guaranteed NOT to appear in normal assistant output. +BLOCKED_PHRASE = "XSECRETBLOCKTESTPHRASEX" + + +class PhraseBlockingGuardrail(CustomGuardrail): + """Blocks any message containing BLOCKED_PHRASE.""" + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + for text in inputs.get("texts", []): + if BLOCKED_PHRASE in text: + raise ValueError( + "Content blocked: contains forbidden test phrase." + ) + return inputs + + +def _make_guardrail(event_hook=GuardrailEventHooks.pre_call): + return PhraseBlockingGuardrail( + guardrail_name="integration-test-guard", + event_hook=event_hook, + default_on=True, + ) + + +async def _wait_for_event( + client_events: List[dict], event_type: str, timeout: float = 15.0 +) -> dict: + """Poll client_events list until an event with matching type appears.""" + deadline = asyncio.get_event_loop().time() + timeout + while asyncio.get_event_loop().time() < deadline: + matching = [e for e in client_events if e.get("type") == event_type] + if matching: + return matching[0] + await asyncio.sleep(0.05) + raise TimeoutError( + f"Timed out waiting for '{event_type}'. Got so far: {[e.get('type') for e in client_events]}" + ) + + +async def _build_streaming(client_events: List[dict], backend_ws, request_data=None): + """Create a RealTimeStreaming with a mock client WebSocket that captures events.""" + client_ws = MagicMock() + input_queue: asyncio.Queue = asyncio.Queue() + + async def send_text(data: str): + client_events.append(json.loads(data)) + + client_ws.send_text = send_text + client_ws.receive_text = input_queue.get + + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + logging_obj.model_call_details = {} + + streaming = RealTimeStreaming( + websocket=client_ws, + backend_ws=backend_ws, + logging_obj=logging_obj, + request_data=request_data or {"guardrails": ["integration-test-guard"]}, + ) + return streaming, input_queue + + +@pytest.mark.asyncio +async def test_text_message_blocked_by_guardrail_no_ai_response(): + """ + Send a text message containing the blocked phrase. + Guardrail must: + - Send error event (guardrail_violation) to client. + - Send response.audio_transcript.delta with the block message to client. + - NOT forward response.create to OpenAI (no AI response). + """ + import websockets + + guardrail = _make_guardrail(GuardrailEventHooks.pre_call) + litellm.callbacks = [guardrail] + + client_events: List[dict] = [] + + try: + async with websockets.connect( + OPENAI_REALTIME_URL, + additional_headers={ + "Authorization": f"Bearer {OPENAI_API_KEY}", + "OpenAI-Beta": "realtime=v1", + }, + ) as backend_ws: + streaming, input_queue = await _build_streaming(client_events, backend_ws) + + # Start backend -> client forwarding + backend_task = asyncio.create_task( + streaming.backend_to_client_send_messages() + ) + # Start client -> backend forwarding (reads from input_queue) + client_task = asyncio.create_task(streaming.client_ack_messages()) + + try: + # Wait until session is ready + await _wait_for_event(client_events, "session.created", timeout=15) + + # Send the blocked message + response.create + blocked_item = json.dumps( + { + "type": "conversation.item.create", + "item": { + "role": "user", + "content": [ + { + "type": "input_text", + "text": f"Hello {BLOCKED_PHRASE}", + } + ], + }, + } + ) + await input_queue.put(blocked_item) + # Give guardrail time to process before the follow-up response.create + await asyncio.sleep(0.3) + await input_queue.put(json.dumps({"type": "response.create"})) + + # Allow time for guardrail round-trip + await asyncio.sleep(3.0) + + finally: + backend_task.cancel() + client_task.cancel() + await asyncio.gather(backend_task, client_task, return_exceptions=True) + + # --- Assertions --- + event_types = [e.get("type") for e in client_events] + + # 1. Must have received guardrail error + error_events = [e for e in client_events if e.get("type") == "error"] + assert len(error_events) >= 1, ( + f"Expected at least one error event but got: {event_types}" + ) + assert error_events[0]["error"]["type"] == "guardrail_violation", ( + f"Wrong error type: {error_events[0]}" + ) + + # 2. Must have the guardrail message surfaced as an AI transcript delta + transcript_deltas = [ + e + for e in client_events + if e.get("type") == "response.audio_transcript.delta" + ] + assert len(transcript_deltas) >= 1, ( + f"Expected guardrail message in transcript delta, got: {event_types}" + ) + + # 3. No real AI response should have been generated - response.done would only + # appear if we sent a response.create and OpenAI replied. We allow it in the + # synthetic form (empty output=[]) but NOT with actual AI content. + done_events = [e for e in client_events if e.get("type") == "response.done"] + for done in done_events: + output = done.get("response", {}).get("output", []) + ai_texts = [ + c.get("text", "") or c.get("transcript", "") + for item in output + for c in item.get("content", []) + ] + real_ai_text = " ".join(ai_texts).strip() + assert real_ai_text == "", ( + f"AI responded with real content even though message was blocked: {real_ai_text!r}" + ) + + finally: + litellm.callbacks = [] + + +@pytest.mark.asyncio +async def test_voice_transcript_blocked_by_guardrail(): + """ + Simulate a backend-side voice transcription event containing the blocked phrase. + Guardrail must block it - no response.create sent to OpenAI. + """ + from websockets.exceptions import ConnectionClosed + + guardrail = _make_guardrail(GuardrailEventHooks.realtime_input_transcription) + litellm.callbacks = [guardrail] + + client_events: List[dict] = [] + + # Build the transcript event that would come from the OpenAI backend + transcript_event = json.dumps( + { + "type": "conversation.item.input_audio_transcription.completed", + "transcript": f"This is {BLOCKED_PHRASE} in my voice message", + "item_id": "item_integ_test", + } + ).encode() + + # Mock backend that delivers the transcript then closes + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[ + transcript_event, + ConnectionClosed(None, None), + ] + ) + backend_ws.send = AsyncMock() + + try: + streaming, _ = await _build_streaming(client_events, backend_ws) + await streaming.backend_to_client_send_messages() + + event_types = [e.get("type") for e in client_events] + + # 1. Error event must be sent to client + error_events = [e for e in client_events if e.get("type") == "error"] + assert len(error_events) >= 1, ( + f"Expected guardrail error event, got: {event_types}" + ) + assert error_events[0]["error"]["type"] == "guardrail_violation" + + # 2. response.create must NOT have been sent to backend + sent_to_backend = [ + json.loads(c.args[0]) + for c in backend_ws.send.call_args_list + if c.args and isinstance(c.args[0], str) + ] + response_creates = [ + e for e in sent_to_backend if e.get("type") == "response.create" + ] + assert len(response_creates) == 0, ( + f"Guardrail should have stopped response.create, got: {sent_to_backend}" + ) + + # 3. Guardrail message surfaced as AI transcript delta + transcript_deltas = [ + e + for e in client_events + if e.get("type") == "response.audio_transcript.delta" + ] + assert len(transcript_deltas) >= 1, ( + f"Expected guardrail message in transcript delta, got: {event_types}" + ) + + finally: + litellm.callbacks = [] + + +@pytest.mark.asyncio +async def test_clean_text_message_passes_through_to_openai(): + """ + A clean message (no blocked phrase) must pass the guardrail and result in a real + AI response from OpenAI (response.done with non-empty output). + """ + import websockets + + guardrail = _make_guardrail(GuardrailEventHooks.pre_call) + litellm.callbacks = [guardrail] + + client_events: List[dict] = [] + + try: + async with websockets.connect( + OPENAI_REALTIME_URL, + additional_headers={ + "Authorization": f"Bearer {OPENAI_API_KEY}", + "OpenAI-Beta": "realtime=v1", + }, + ) as backend_ws: + streaming, input_queue = await _build_streaming(client_events, backend_ws) + + backend_task = asyncio.create_task( + streaming.backend_to_client_send_messages() + ) + client_task = asyncio.create_task(streaming.client_ack_messages()) + + try: + await _wait_for_event(client_events, "session.created", timeout=15) + + # Send a clean message + clean_item = json.dumps( + { + "type": "conversation.item.create", + "item": { + "role": "user", + "content": [ + {"type": "input_text", "text": "Reply with just: OK"} + ], + }, + } + ) + await input_queue.put(clean_item) + await asyncio.sleep(0.1) + await input_queue.put(json.dumps({"type": "response.create"})) + + # Wait for OpenAI to respond + await _wait_for_event(client_events, "response.done", timeout=30) + + finally: + backend_task.cancel() + client_task.cancel() + await asyncio.gather(backend_task, client_task, return_exceptions=True) + + # No guardrail error should have been sent + error_events = [e for e in client_events if e.get("type") == "error"] + guardrail_errors = [ + e for e in error_events if e.get("error", {}).get("type") == "guardrail_violation" + ] + assert len(guardrail_errors) == 0, ( + f"Clean message should not trigger guardrail, got: {guardrail_errors}" + ) + + # AI response must be present + done_events = [e for e in client_events if e.get("type") == "response.done"] + assert len(done_events) >= 1, ( + f"Expected response.done from OpenAI, got: {[e.get('type') for e in client_events]}" + ) + + finally: + litellm.callbacks = [] diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index bcda3c7bfac..11d6bb028d8 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -637,9 +637,10 @@ async def test_realtime_text_input_guardrail_uses_pre_call_mode(): assert streaming._has_realtime_guardrails() is True, ( "pre_call guardrail should be recognized as a realtime guardrail" ) - # pre_call guardrail should NOT trigger the audio/VAD session.update injection - assert streaming._has_audio_transcription_guardrails() is False, ( - "pre_call guardrail should not trigger audio transcription guardrail path" + # pre_call guardrail SHOULD trigger the audio/VAD session.update injection so + # that the LLM does not auto-respond before the guardrail can check the transcript. + assert streaming._has_audio_transcription_guardrails() is True, ( + "pre_call guardrail should trigger audio transcription guardrail path" ) litellm.callbacks = [] # cleanup @@ -711,10 +712,11 @@ async def test_realtime_session_created_injects_session_update_for_audio_guardra @pytest.mark.asyncio -async def test_realtime_session_created_no_injection_for_pre_call_only(): +async def test_realtime_session_created_injects_session_update_for_pre_call_guardrail(): """ - Test that when only a pre_call guardrail is configured (no audio transcription), - session.created does NOT trigger the session.update injection. + Test that when a pre_call guardrail is configured, session.created triggers the + session.update injection (create_response: false) so the LLM does not auto-respond + before the guardrail can check the voice transcript. """ import litellm from litellm.integrations.custom_guardrail import CustomGuardrail @@ -751,14 +753,15 @@ async def test_realtime_session_created_no_injection_for_pre_call_only(): streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) await streaming.backend_to_client_send_messages() - # No session.update should be injected + # session.update SHOULD be injected so the LLM waits for guardrail approval sent_to_backend = [ json.loads(c.args[0]) for c in backend_ws.send.call_args_list if c.args ] session_updates = [e for e in sent_to_backend if e.get("type") == "session.update"] - assert len(session_updates) == 0, ( - f"pre_call guardrail should NOT inject session.update, got: {sent_to_backend}" + assert len(session_updates) == 1, ( + f"pre_call guardrail should inject session.update to gate audio responses, got: {sent_to_backend}" ) + assert session_updates[0]["session"]["turn_detection"]["create_response"] is False litellm.callbacks = [] # cleanup From 71c3503e57c27ed929e63a36d4f6314ff9f441a6 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 17:21:43 -0800 Subject: [PATCH 21/54] Revert "[Feature] Add /public/supported_endpoints endpoint" --- .../RELEASE_NOTES_GENERATION_INSTRUCTIONS.md | 4 +- .../public_endpoints/public_endpoints.py | 68 ------------------- .../public_endpoints/public_endpoints.py | 17 ----- ...rt.json => provider_endpoints_support.json | 0 .../check_endpoint_coverage.py | 2 +- .../check_provider_folders_documented.py | 2 +- .../public_endpoints/test_public_endpoints.py | 48 ------------- 7 files changed, 4 insertions(+), 137 deletions(-) rename litellm/proxy/public_endpoints/provider_endpoints_support.json => provider_endpoints_support.json (100%) diff --git a/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md b/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md index b6665a76773..ab2cf334459 100644 --- a/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md +++ b/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md @@ -404,7 +404,7 @@ This release has a known issue... - **New Providers** - Provider name, supported endpoints, description - **New LLM API Endpoints** (optional) - Endpoint, method, description, documentation link - Only include major new provider integrations, not minor provider updates -- **IMPORTANT**: When adding new providers, also update `provider_endpoints_support.json` (see Section 13) +- **IMPORTANT**: When adding new providers, also update `provider_endpoints_support.json` in the repository root (see Section 13) ### 12. Section Header Counts @@ -442,7 +442,7 @@ This release has a known issue... ### 13. Update provider_endpoints_support.json -**When adding new providers or endpoints, you MUST also update `litellm/proxy/public_endpoints/provider_endpoints_support.json`.** +**When adding new providers or endpoints, you MUST also update `provider_endpoints_support.json` in the repository root.** This file tracks which endpoints are supported by each LiteLLM provider and is used to generate documentation. diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 54fc753e768..29c9cb571ca 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -1,6 +1,5 @@ import json import os -import re from typing import List import litellm @@ -24,16 +23,11 @@ from litellm.types.proxy.public_endpoints.public_endpoints import ( AgentCreateInfo, ProviderCreateInfo, PublicModelHubInfo, - SupportedEndpointInfo, - SupportedEndpointsResponse, - SupportedProviderInfo, ) from litellm.types.utils import LlmProviders router = APIRouter() -_supported_endpoints_cache: SupportedEndpointsResponse | None = None - @router.get( "/public/model_hub", @@ -231,68 +225,6 @@ async def get_litellm_blog_posts(): return BlogPostsResponse(posts=posts) -@router.get( - "/public/supported_endpoints", - tags=["public", "providers"], - response_model=SupportedEndpointsResponse, -) -async def get_provider_supported_endpoints() -> SupportedEndpointsResponse: - """ - Return all supported endpoints and which providers support them. - - Reads from provider_endpoints_support.json at the repo root. - Result is cached for the lifetime of the process. - """ - global _supported_endpoints_cache - if _supported_endpoints_cache is not None: - return _supported_endpoints_cache - - provider_endpoints_support_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))), - "proxy", - "public_endpoints", - "provider_endpoints_support.json", - ) - - with open(provider_endpoints_support_path, "r") as f: - data = json.load(f) - - schema_endpoints = data["_schema"]["provider_slug"]["endpoints"] - - endpoints = [] - for key, description in schema_endpoints.items(): - path_match = re.search(r"(/[\w/{}.()*-]+)", description) - endpoint_path = path_match.group(1) if path_match else f"/{key}" - display_name = key.replace("_", " ").title() - endpoints.append( - SupportedEndpointInfo( - key=key, - display_name=display_name, - endpoint=endpoint_path, - ) - ) - - providers = [] - for slug, provider_data in data["providers"].items(): - supported = [ - endpoint_key - for endpoint_key, supported in provider_data["endpoints"].items() - if supported - ] - providers.append( - SupportedProviderInfo( - slug=slug, - display_name=provider_data["display_name"], - supported=supported, - ) - ) - - _supported_endpoints_cache = SupportedEndpointsResponse( - endpoints=endpoints, providers=providers - ) - return _supported_endpoints_cache - - @router.get( "/public/agents/fields", tags=["public", "[beta] Agents"], diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index a167eeec6a2..57d68771c7f 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -52,20 +52,3 @@ class AgentCreateInfo(BaseModel): credential_fields: List[AgentCredentialField] litellm_params_template: Optional[Dict[str, str]] = None model_template: Optional[str] = None - - -class SupportedEndpointInfo(BaseModel): - key: str - display_name: str - endpoint: str - - -class SupportedProviderInfo(BaseModel): - slug: str - display_name: str - supported: List[str] - - -class SupportedEndpointsResponse(BaseModel): - endpoints: List[SupportedEndpointInfo] - providers: List[SupportedProviderInfo] diff --git a/litellm/proxy/public_endpoints/provider_endpoints_support.json b/provider_endpoints_support.json similarity index 100% rename from litellm/proxy/public_endpoints/provider_endpoints_support.json rename to provider_endpoints_support.json diff --git a/tests/code_coverage_tests/check_endpoint_coverage.py b/tests/code_coverage_tests/check_endpoint_coverage.py index 25f181aa3a5..2d46d1ab469 100644 --- a/tests/code_coverage_tests/check_endpoint_coverage.py +++ b/tests/code_coverage_tests/check_endpoint_coverage.py @@ -99,7 +99,7 @@ def extract_endpoints_from_sidebars() -> Dict[str, str]: def load_provider_endpoints_file() -> Dict: """Load the provider_endpoints_support.json file.""" repo_root = get_repo_root() - file_path = repo_root / "litellm" / "proxy" / "public_endpoints" / "provider_endpoints_support.json" + file_path = repo_root / "provider_endpoints_support.json" if not file_path.exists(): print( diff --git a/tests/code_coverage_tests/check_provider_folders_documented.py b/tests/code_coverage_tests/check_provider_folders_documented.py index d5f7c7f7a07..60afc55331f 100644 --- a/tests/code_coverage_tests/check_provider_folders_documented.py +++ b/tests/code_coverage_tests/check_provider_folders_documented.py @@ -65,7 +65,7 @@ def get_llm_provider_folders() -> Set[str]: def load_provider_endpoints_file() -> Dict: """Load the provider_endpoints_support.json file.""" repo_root = get_repo_root() - file_path = repo_root / "litellm" / "proxy" / "public_endpoints" / "provider_endpoints_support.json" + file_path = repo_root / "provider_endpoints_support.json" if not file_path.exists(): print( diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 0b21bc2636c..5f5e2cf1ff8 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -87,54 +87,6 @@ def test_get_litellm_model_cost_map_returns_cost_map(): assert "input_cost_per_token" in sample_model_data or "output_cost_per_token" in sample_model_data -def test_get_provider_supported_endpoints(): - """Test /public/supported_endpoints returns correct structure with endpoints and providers.""" - app = FastAPI() - app.include_router(router) - client = TestClient(app) - - response = client.get("/public/supported_endpoints") - - assert response.status_code == 200 - data = response.json() - - # Check top-level structure - assert "endpoints" in data - assert "providers" in data - assert isinstance(data["endpoints"], list) - assert isinstance(data["providers"], list) - - # Verify endpoints structure - assert len(data["endpoints"]) > 0 - for endpoint in data["endpoints"]: - assert "key" in endpoint - assert "display_name" in endpoint - assert "endpoint" in endpoint - assert isinstance(endpoint["key"], str) - assert isinstance(endpoint["display_name"], str) - assert endpoint["endpoint"].startswith("/") - - # Verify providers structure - assert len(data["providers"]) > 0 - for provider in data["providers"]: - assert "slug" in provider - assert "display_name" in provider - assert "supported" in provider - assert isinstance(provider["slug"], str) - assert isinstance(provider["display_name"], str) - assert isinstance(provider["supported"], list) - - # Verify some expected endpoints exist - endpoint_keys = {e["key"] for e in data["endpoints"]} - assert "chat_completions" in endpoint_keys - assert "embeddings" in endpoint_keys - assert "responses" in endpoint_keys - - # Verify some expected providers exist - provider_slugs = {p["slug"] for p in data["providers"]} - assert "openai" in provider_slugs - - def test_watsonx_provider_fields(): """Test that Watsonx provider has all required credential fields including multiple auth options.""" app = FastAPI() From df36845839378b3c8bff0cffd69aac5aa4bb2e1d Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Thu, 26 Feb 2026 17:39:01 -0800 Subject: [PATCH 22/54] fix: remove cache eviction close that kills in-use httpx clients --- litellm/caching/llm_caching_handler.py | 19 ---------- .../caching/test_redis_connection_pool.py | 36 ------------------- 2 files changed, 55 deletions(-) diff --git a/litellm/caching/llm_caching_handler.py b/litellm/caching/llm_caching_handler.py index 5dc16a224c7..16eb824f4c9 100644 --- a/litellm/caching/llm_caching_handler.py +++ b/litellm/caching/llm_caching_handler.py @@ -8,25 +8,6 @@ from .in_memory_cache import InMemoryCache class LLMClientCache(InMemoryCache): - def _remove_key(self, key: str) -> None: - """Close async clients before evicting them to prevent connection pool leaks.""" - value = self.cache_dict.get(key) - super()._remove_key(key) - if value is not None: - close_fn = getattr(value, "aclose", None) or getattr( - value, "close", None - ) - if close_fn and asyncio.iscoroutinefunction(close_fn): - try: - asyncio.get_running_loop().create_task(close_fn()) - except RuntimeError: - pass - elif close_fn and callable(close_fn): - try: - close_fn() - except Exception: - pass - def update_cache_key_with_event_loop(self, key): """ Add the event loop to the cache key, to prevent event loop closed errors. diff --git a/tests/test_litellm/caching/test_redis_connection_pool.py b/tests/test_litellm/caching/test_redis_connection_pool.py index b8922846e82..f6e429ceff9 100644 --- a/tests/test_litellm/caching/test_redis_connection_pool.py +++ b/tests/test_litellm/caching/test_redis_connection_pool.py @@ -4,14 +4,12 @@ Regression tests for Redis connection pool leak fixes (RC1-RC5). Tests are pure unit tests — no Redis server required. """ -import asyncio from unittest.mock import AsyncMock, MagicMock, patch import pytest import redis.asyncio as async_redis from litellm._redis import get_redis_async_client, get_redis_connection_pool -from litellm.caching.llm_caching_handler import LLMClientCache def test_url_config_uses_passed_pool(): @@ -131,37 +129,3 @@ async def test_disconnect_idempotent(): await cache.disconnect() # should not raise -@pytest.mark.asyncio -async def test_eviction_calls_aclose(): - """When an async client is evicted from LLMClientCache, its aclose() - should be scheduled via create_task.""" - cache = LLMClientCache(max_size_in_memory=2, default_ttl=600) - - client = AsyncMock() - client.aclose = AsyncMock() - - cache.set_cache(key="client-0", value=client) - cache.set_cache(key="filler", value="x") - # Third insert triggers eviction of client-0 - cache.set_cache(key="trigger", value="y") - - # Let the scheduled task run - await asyncio.sleep(0.05) - - assert client.aclose.await_count > 0 - - -@pytest.mark.asyncio -async def test_eviction_non_closeable_safe(): - """Evicting plain values (strings, dicts, ints) should not crash.""" - cache = LLMClientCache(max_size_in_memory=2, default_ttl=600) - - cache.set_cache(key="str-val", value="hello") - cache.set_cache(key="dict-val", value={"foo": "bar"}) - # This evicts "str-val" — should not raise - cache.set_cache(key="int-val", value=42) - - await asyncio.sleep(0.05) - - # If we got here without exception, the test passes - assert cache.get_cache(key="int-val") == 42 From 86b2efd67a95933eee4f7d2b307dc2849222a01b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 18:17:37 -0800 Subject: [PATCH 23/54] [Feature] Add /public/endpoints endpoint for provider endpoint support Add new /public/endpoints endpoint that returns which providers support each LiteLLM endpoint (e.g., chat_completions, embeddings). The endpoint reads from a local backup JSON file bundled with the package, caches the result in-process, and transforms the raw provider-centric data into an endpoint-centric response format. Changes: - Add litellm/provider_endpoints_support_backup.json (copy of root source file) - Add Pydantic response models (EndpointProvider, SupportedEndpoint, SupportedEndpointsResponse) - Add /public/endpoints route with transformation and caching logic - Add 16 comprehensive tests covering HTTP layer and transformation functions Co-Authored-By: Claude Haiku 4.5 --- .../provider_endpoints_support_backup.json | 2748 +++++++++++++++++ .../public_endpoints/public_endpoints.py | 116 +- .../public_endpoints/public_endpoints.py | 16 + .../public_endpoints/test_public_endpoints.py | 161 + 4 files changed, 3035 insertions(+), 6 deletions(-) create mode 100644 litellm/provider_endpoints_support_backup.json diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json new file mode 100644 index 00000000000..8834d8b19c0 --- /dev/null +++ b/litellm/provider_endpoints_support_backup.json @@ -0,0 +1,2748 @@ +{ + "_comment": "This file defines which endpoints are supported by each LiteLLM provider", + "_schema": { + "provider_slug": { + "display_name": "Display name shown in README (e.g., 'OpenAI (`openai`)')", + "url": "Link to provider documentation", + "endpoints": { + "chat_completions": "Supports /chat/completions endpoint", + "messages": "Supports /messages endpoint (Anthropic format)", + "responses": "Supports /responses endpoint (OpenAI/Anthropic unified)", + "embeddings": "Supports /embeddings endpoint", + "image_generations": "Supports /image/generations endpoint", + "audio_transcriptions": "Supports /audio/transcriptions endpoint", + "audio_speech": "Supports /audio/speech endpoint", + "moderations": "Supports /moderations endpoint", + "batches": "Supports /batches endpoint", + "rerank": "Supports /rerank endpoint", + "ocr": "Supports /ocr endpoint", + "search": "Supports /search endpoint", + "skills": "Supports /skills endpoint", + "interactions": "Supports /interactions endpoint (Google AI Interactions API)", + "a2a_(Agent Gateway)": "Supports /a2a/{agent}/message/send endpoint (A2A Protocol)", + "container": "Supports OpenAI's /containers endpoint", + "container_file": "Supports OpenAI's /containers/{id}/files endpoint", + "compact": "Supports /responses/compact endpoint", + "files": "Supports /files endpoint for file operations", + "image_edits": "Supports /images/edits endpoint for image editing", + "vector_stores_create": "Supports creating a new vector store via /vector_stores endpoint", + "vector_stores_search": "Supports searching a vector store via /vector_stores/{id}/search endpoint", + "video_generations": "Supports /videos/generations endpoint for video generation" + } + } + }, + "providers": { + "a2a": { + "display_name": "A2A (Agent-to-Agent) (`a2a`)", + "url": "https://docs.litellm.ai/docs/providers/a2a", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "abliteration": { + "display_name": "Abliteration (`abliteration`)", + "url": "https://docs.litellm.ai/docs/providers/abliteration", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "aiml": { + "display_name": "AI/ML API (`aiml`)", + "url": "https://docs.litellm.ai/docs/providers/aiml", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ai21": { + "display_name": "AI21 (`ai21`)", + "url": "https://docs.litellm.ai/docs/providers/ai21", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ai21_chat": { + "display_name": "AI21 Chat (`ai21_chat`)", + "url": "https://docs.litellm.ai/docs/providers/ai21", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "amazon_nova": { + "display_name": "Amazon Nova (`amazon_nova`)", + "url": "https://docs.litellm.ai/docs/providers/amazon_nova", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "anthropic": { + "display_name": "Anthropic (`anthropic`)", + "url": "https://docs.litellm.ai/docs/providers/anthropic", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": true, + "rerank": false, + "skills": true, + "a2a": true, + "interactions": true, + "count_tokens": true + } + }, + "anthropic_text": { + "display_name": "Anthropic Text (`anthropic_text`)", + "url": "https://docs.litellm.ai/docs/providers/anthropic", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": true, + "rerank": false, + "skills": true, + "a2a": true, + "interactions": true + } + }, + "apertis": { + "display_name": "Apertis (`apertis`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "assemblyai": { + "display_name": "AssemblyAI (`assemblyai`)", + "url": "https://docs.litellm.ai/docs/pass_through/assembly_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "auto_router": { + "display_name": "Auto Router (`auto_router`)", + "url": "https://docs.litellm.ai/docs/proxy/auto_routing", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "bedrock": { + "display_name": "AWS - Bedrock (`bedrock`)", + "url": "https://docs.litellm.ai/docs/providers/bedrock", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": true, + "a2a": true, + "interactions": true, + "bedrock_invoke": true, + "bedrock_converse": true, + "vector_stores_search": true, + "count_tokens": true, + "rag_ingest": true, + "rag_query": true + } + }, + "s3_vectors": { + "display_name": "AWS S3 Vectors (`s3_vectors`)", + "url": "https://docs.litellm.ai/docs/providers/s3_vectors", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false, + "vector_stores_create": true, + "vector_stores_search": true + } + }, + "sagemaker": { + "display_name": "AWS - Sagemaker (`sagemaker`)", + "url": "https://docs.litellm.ai/docs/providers/aws_sagemaker", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "aws_polly": { + "display_name": "AWS - Polly (`aws_polly`)", + "url": "https://docs.litellm.ai/docs/providers/aws_polly", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": true, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "azure": { + "display_name": "Azure (`azure`)", + "url": "https://docs.litellm.ai/docs/providers/azure", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "a2a": true, + "interactions": true, + "vector_stores_search": true, + "assistants": true, + "fine_tuning": true, + "text_completion": true + } + }, + "azure_ai": { + "display_name": "Azure AI (`azure_ai`)", + "url": "https://docs.litellm.ai/docs/providers/azure_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "image_edits": true, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "ocr": true, + "a2a": true, + "interactions": true, + "vector_stores_create": true, + "vector_stores_search": true + } + }, + "azure_ai/doc-intelligence": { + "display_name": "Azure AI Document Intelligence (`azure_ai/doc-intelligence`)", + "url": "https://docs.litellm.ai/docs/providers/azure_document_intelligence", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "ocr": true + } + }, + "azure_ai/agents": { + "display_name": "Azure AI Foundry Agents (`azure_ai/agents`)", + "url": "https://docs.litellm.ai/docs/providers/azure_ai_agents", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "azure_text": { + "display_name": "Azure Text (`azure_text`)", + "url": "https://docs.litellm.ai/docs/providers/azure", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "baseten": { + "display_name": "Baseten (`baseten`)", + "url": "https://docs.litellm.ai/docs/providers/baseten", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "bytez": { + "display_name": "Bytez (`bytez`)", + "url": "https://docs.litellm.ai/docs/providers/bytez", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "cerebras": { + "display_name": "Cerebras (`cerebras`)", + "url": "https://docs.litellm.ai/docs/providers/cerebras", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "chutes": { + "display_name": "Chutes (`chutes`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "clarifai": { + "display_name": "Clarifai (`clarifai`)", + "url": "https://docs.litellm.ai/docs/providers/clarifai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "cloudflare": { + "display_name": "Cloudflare AI Workers (`cloudflare`)", + "url": "https://docs.litellm.ai/docs/providers/cloudflare_workers", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "codestral": { + "display_name": "Codestral (`codestral`)", + "url": "https://docs.litellm.ai/docs/providers/codestral", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "cohere": { + "display_name": "Cohere (`cohere`)", + "url": "https://docs.litellm.ai/docs/providers/cohere", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": true, + "a2a": true, + "interactions": true + } + }, + "cohere_chat": { + "display_name": "Cohere Chat (`cohere_chat`)", + "url": "https://docs.litellm.ai/docs/providers/cohere", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "cometapi": { + "display_name": "CometAPI (`cometapi`)", + "url": "https://docs.litellm.ai/docs/providers/cometapi", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "compactifai": { + "display_name": "CompactifAI (`compactifai`)", + "url": "https://docs.litellm.ai/docs/providers/compactifai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "custom": { + "display_name": "Custom (`custom`)", + "url": "https://docs.litellm.ai/docs/providers/custom_llm_server", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "custom_openai": { + "display_name": "Custom OpenAI (`custom_openai`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "dashscope": { + "display_name": "Dashscope (`dashscope`)", + "url": "https://docs.litellm.ai/docs/providers/dashscope", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "databricks": { + "display_name": "Databricks (`databricks`)", + "url": "https://docs.litellm.ai/docs/providers/databricks", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "dataforseo": { + "display_name": "DataForSEO (`dataforseo`)", + "url": "https://docs.litellm.ai/docs/search/dataforseo", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "datarobot": { + "display_name": "DataRobot (`datarobot`)", + "url": "https://docs.litellm.ai/docs/providers/datarobot", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "deepgram": { + "display_name": "Deepgram (`deepgram`)", + "url": "https://docs.litellm.ai/docs/providers/deepgram", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "deepinfra": { + "display_name": "DeepInfra (`deepinfra`)", + "url": "https://docs.litellm.ai/docs/providers/deepinfra", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "deepseek": { + "display_name": "Deepseek (`deepseek`)", + "url": "https://docs.litellm.ai/docs/providers/deepseek", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "duckduckgo": { + "display_name": "DuckDuckGo (`duckduckgo`)", + "url": "https://docs.litellm.ai/docs/search/duckduckgo", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "elevenlabs": { + "display_name": "ElevenLabs (`elevenlabs`)", + "url": "https://docs.litellm.ai/docs/providers/elevenlabs", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": true, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "exa_ai": { + "display_name": "Exa AI (`exa_ai`)", + "url": "https://docs.litellm.ai/docs/search/exa_ai", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "brave": { + "display_name": "Brave Search (`brave`)", + "url": "https://docs.litellm.ai/docs/search/brave", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "empower": { + "display_name": "Empower (`empower`)", + "url": "https://docs.litellm.ai/docs/providers/empower", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "fal_ai": { + "display_name": "Fal AI (`fal_ai`)", + "url": "https://docs.litellm.ai/docs/providers/fal_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "featherless_ai": { + "display_name": "Featherless AI (`featherless_ai`)", + "url": "https://docs.litellm.ai/docs/providers/featherless_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "fireworks_ai": { + "display_name": "Fireworks AI (`fireworks_ai`)", + "url": "https://docs.litellm.ai/docs/providers/fireworks_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": true, + "a2a": true, + "interactions": true + } + }, + "firecrawl": { + "display_name": "Firecrawl (`firecrawl`)", + "url": "https://docs.litellm.ai/docs/search/firecrawl", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "linkup": { + "display_name": "Linkup (`linkup`)", + "url": "https://docs.litellm.ai/docs/search/linkup", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "friendliai": { + "display_name": "FriendliAI (`friendliai`)", + "url": "https://docs.litellm.ai/docs/providers/friendliai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "galadriel": { + "display_name": "Galadriel (`galadriel`)", + "url": "https://docs.litellm.ai/docs/providers/galadriel", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "github_copilot": { + "display_name": "GitHub Copilot (`github_copilot`)", + "url": "https://docs.litellm.ai/docs/providers/github_copilot", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "chatgpt": { + "display_name": "ChatGPT Subscription (`chatgpt`)", + "url": "https://docs.litellm.ai/docs/providers/chatgpt", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, + "github": { + "display_name": "GitHub Models (`github`)", + "url": "https://docs.litellm.ai/docs/providers/github", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "gmi": { + "display_name": "GMI Cloud (`gmi`)", + "url": "https://docs.litellm.ai/docs/providers/gmi_cloud", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "vertex_ai": { + "display_name": "Google - Vertex AI (`vertex_ai`)", + "url": "https://docs.litellm.ai/docs/providers/vertex", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": true, + "moderations": false, + "batches": false, + "rerank": false, + "ocr": true, + "a2a": true, + "interactions": true, + "vector_stores_search": true, + "count_tokens": true, + "fine_tuning": true, + "rag_ingest": true, + "rag_query": true, + "generateContent": true, + "realtime": true + } + }, + "gemini": { + "display_name": "Google AI Studio - Gemini (`gemini`)", + "url": "https://docs.litellm.ai/docs/providers/gemini", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "interactions": true, + "a2a": true, + "vector_stores_search": true, + "count_tokens": true, + "rag_ingest": true, + "realtime": true, + "generateContent": true + } + }, + "gradient_ai": { + "display_name": "GradientAI (`gradient_ai`)", + "url": "https://docs.litellm.ai/docs/providers/gradient_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "groq": { + "display_name": "Groq AI (`groq`)", + "url": "https://docs.litellm.ai/docs/providers/groq", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "heroku": { + "display_name": "Heroku (`heroku`)", + "url": "https://docs.litellm.ai/docs/providers/heroku", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "hosted_vllm": { + "display_name": "Hosted VLLM (`hosted_vllm`)", + "url": "https://docs.litellm.ai/docs/providers/vllm", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": true, + "files": true, + "rerank": true, + "a2a": true, + "interactions": true + } + }, + "huggingface": { + "display_name": "Huggingface (`huggingface`)", + "url": "https://docs.litellm.ai/docs/providers/huggingface", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": true, + "a2a": true, + "interactions": true + } + }, + "hyperbolic": { + "display_name": "Hyperbolic (`hyperbolic`)", + "url": "https://docs.litellm.ai/docs/providers/hyperbolic", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "watsonx": { + "display_name": "IBM - Watsonx.ai (`watsonx`)", + "url": "https://docs.litellm.ai/docs/providers/watsonx", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "infinity": { + "display_name": "Infinity (`infinity`)", + "url": "https://docs.litellm.ai/docs/providers/infinity", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "jina_ai": { + "display_name": "Jina AI (`jina_ai`)", + "url": "https://docs.litellm.ai/docs/providers/jina_ai", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "lambda_ai": { + "display_name": "Lambda AI (`lambda_ai`)", + "url": "https://docs.litellm.ai/docs/providers/lambda_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "lemonade": { + "display_name": "Lemonade (`lemonade`)", + "url": "https://docs.litellm.ai/docs/providers/lemonade", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "litellm_proxy": { + "display_name": "LiteLLM Proxy (`litellm_proxy`)", + "url": "https://docs.litellm.ai/docs/providers/litellm_proxy", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "llamafile": { + "display_name": "Llamafile (`llamafile`)", + "url": "https://docs.litellm.ai/docs/providers/llamafile", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "lm_studio": { + "display_name": "LM Studio (`lm_studio`)", + "url": "https://docs.litellm.ai/docs/providers/lm_studio", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "maritalk": { + "display_name": "Maritalk (`maritalk`)", + "url": "https://docs.litellm.ai/docs/providers/maritalk", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "meta_llama": { + "display_name": "Meta - Llama API (`meta_llama`)", + "url": "https://docs.litellm.ai/docs/providers/meta_llama", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "mistral": { + "display_name": "Mistral AI API (`mistral`)", + "url": "https://docs.litellm.ai/docs/providers/mistral", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "ocr": true, + "a2a": true, + "interactions": true + } + }, + "moonshot": { + "display_name": "Moonshot (`moonshot`)", + "url": "https://docs.litellm.ai/docs/providers/moonshot", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "docker_model_runner": { + "display_name": "Docker Model Runner (`docker_model_runner`)", + "url": "https://docs.litellm.ai/docs/providers/docker_model_runner", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "morph": { + "display_name": "Morph (`morph`)", + "url": "https://docs.litellm.ai/docs/providers/morph", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "nanogpt": { + "display_name": "NanoGPT (`nanogpt`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "nebius": { + "display_name": "Nebius AI Studio (`nebius`)", + "url": "https://docs.litellm.ai/docs/providers/nebius", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "nlp_cloud": { + "display_name": "NLP Cloud (`nlp_cloud`)", + "url": "https://docs.litellm.ai/docs/providers/nlp_cloud", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "novita": { + "display_name": "Novita AI (`novita`)", + "url": "https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "nscale": { + "display_name": "Nscale (`nscale`)", + "url": "https://docs.litellm.ai/docs/providers/nscale", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "nvidia_nim": { + "display_name": "Nvidia NIM (`nvidia_nim`)", + "url": "https://docs.litellm.ai/docs/providers/nvidia_nim", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "oci": { + "display_name": "OCI (`oci`)", + "url": "https://docs.litellm.ai/docs/providers/oci", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ollama": { + "display_name": "Ollama (`ollama`)", + "url": "https://docs.litellm.ai/docs/providers/ollama", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ollama_chat": { + "display_name": "Ollama Chat (`ollama_chat`)", + "url": "https://docs.litellm.ai/docs/providers/ollama", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "oobabooga": { + "display_name": "Oobabooga (`oobabooga`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "openai": { + "display_name": "OpenAI (`openai`)", + "url": "https://docs.litellm.ai/docs/providers/openai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "container": true, + "compact": true, + "a2a": true, + "interactions": true, + "vector_store_files": true, + "vector_stores_create": true, + "vector_stores_search": true, + "assistants": true, + "container_files": true, + "fine_tuning": true, + "image_variations": true, + "rag_ingest": true, + "rag_query": true, + "realtime": true, + "text_completion": true + } + }, + "openai_like": { + "display_name": "OpenAI-like (`openai_like`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "assistants": true + } + }, + "openrouter": { + "display_name": "OpenRouter (`openrouter`)", + "url": "https://docs.litellm.ai/docs/providers/openrouter", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ovhcloud": { + "display_name": "OVHCloud AI Endpoints (`ovhcloud`)", + "url": "https://docs.litellm.ai/docs/providers/ovhcloud", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "parallel_ai": { + "display_name": "Parallel AI (`parallel_ai`)", + "url": "https://docs.litellm.ai/docs/search/parallel_ai", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "perplexity": { + "display_name": "Perplexity AI (`perplexity`)", + "url": "https://docs.litellm.ai/docs/providers/perplexity", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true, + "a2a": true, + "interactions": true + } + }, + "petals": { + "display_name": "Petals (`petals`)", + "url": "https://docs.litellm.ai/docs/providers/petals", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "poe": { + "display_name": "Poe (`poe`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "publicai": { + "display_name": "PublicAI (`publicai`)", + "url": "https://docs.litellm.ai/docs/providers/publicai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "predibase": { + "display_name": "Predibase (`predibase`)", + "url": "https://docs.litellm.ai/docs/providers/predibase", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "recraft": { + "display_name": "Recraft (`recraft`)", + "url": "https://docs.litellm.ai/docs/providers/recraft", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "replicate": { + "display_name": "Replicate (`replicate`)", + "url": "https://docs.litellm.ai/docs/providers/replicate", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "runwayml": { + "display_name": "RunwayML (`runwayml`)", + "url": "https://docs.litellm.ai/docs/providers/runwayml/videos", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": true, + "moderations": false, + "batches": false, + "rerank": false, + "video_generations": true + } + }, + "sagemaker_chat": { + "display_name": "Sagemaker Chat (`sagemaker_chat`)", + "url": "https://docs.litellm.ai/docs/providers/aws_sagemaker", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "searxng": { + "display_name": "SearXNG (`searxng`)", + "url": "https://docs.litellm.ai/docs/search/searxng", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "sambanova": { + "display_name": "Sambanova (`sambanova`)", + "url": "https://docs.litellm.ai/docs/providers/sambanova", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "sap": { + "display_name": "SAP Generative AI Hub (`sap`)", + "url": "https://docs.litellm.ai/docs/providers/sap", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "scaleway": { + "display_name": "Scaleway (`scaleway`)", + "url": "https://docs.litellm.ai/docs/providers/scaleway", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "snowflake": { + "display_name": "Snowflake (`snowflake`)", + "url": "https://docs.litellm.ai/docs/providers/snowflake", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "synthetic": { + "display_name": "Synthetic (`synthetic`)", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "text-completion-codestral": { + "display_name": "Text Completion Codestral (`text-completion-codestral`)", + "url": "https://docs.litellm.ai/docs/providers/codestral", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "text-completion-openai": { + "display_name": "Text Completion OpenAI (`text-completion-openai`)", + "url": "https://docs.litellm.ai/docs/providers/text_completion_openai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "together_ai": { + "display_name": "Together AI (`together_ai`)", + "url": "https://docs.litellm.ai/docs/providers/togetherai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "topaz": { + "display_name": "Topaz (`topaz`)", + "url": "https://docs.litellm.ai/docs/providers/topaz", + "endpoints": { + "image_variations": true + } + }, + "tavily": { + "display_name": "Tavily (`tavily`)", + "url": "https://docs.litellm.ai/docs/search/tavily", + "endpoints": { + "search": true + } + }, + "triton": { + "display_name": "Triton (`triton`)", + "url": "https://docs.litellm.ai/docs/providers/triton-inference-server", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "v0": { + "display_name": "V0 (`v0`)", + "url": "https://docs.litellm.ai/docs/providers/v0", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "vercel_ai_gateway": { + "display_name": "Vercel AI Gateway (`vercel_ai_gateway`)", + "url": "https://docs.litellm.ai/docs/providers/vercel_ai_gateway", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "vllm": { + "display_name": "VLLM (`vllm`)", + "url": "https://docs.litellm.ai/docs/providers/vllm", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": true, + "files": true, + "rerank": true, + "a2a": true, + "interactions": true + } + }, + "volcengine": { + "display_name": "Volcengine (`volcengine`)", + "url": "https://docs.litellm.ai/docs/providers/volcano", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "voyage": { + "display_name": "Voyage AI (`voyage`)", + "url": "https://docs.litellm.ai/docs/providers/voyage", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": true + } + }, + "wandb": { + "display_name": "WandB Inference (`wandb`)", + "url": "https://docs.litellm.ai/docs/providers/wandb_inference", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "watsonx_text": { + "display_name": "Watsonx Text (`watsonx_text`)", + "url": "https://docs.litellm.ai/docs/providers/watsonx", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "xai": { + "display_name": "xAI (`xai`)", + "url": "https://docs.litellm.ai/docs/providers/xai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true, + "realtime": true + } + }, + "xinference": { + "display_name": "Xinference (`xinference`)", + "url": "https://docs.litellm.ai/docs/providers/xinference", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "zai": { + "display_name": "Z.AI (Zhipu AI) (`zai`)", + "url": "https://docs.litellm.ai/docs/providers/zai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ragflow": { + "display_name": "RAGFlow (`ragflow`)", + "url": "https://docs.litellm.ai/docs/providers/ragflow", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "vector_stores_create": true, + "a2a": true, + "interactions": true + } + }, + "cursor": { + "display_name": "Cursor BYOK (`cursor`)", + "url": "https://docs.litellm.ai/docs/providers/cursor", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "langgraph": { + "display_name": "LangGraph (`langgraph`)", + "url": "https://docs.litellm.ai/docs/providers/langgraph", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "vertex_ai/agent_engine": { + "display_name": "Vertex AI Agent Engine (`vertex_ai/agent_engine`)", + "url": "https://docs.litellm.ai/docs/providers/vertex_ai_agent_engine", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "pydantic_ai_agents": { + "display_name": "Pydantic AI Agents (`pydantic_ai_agents`)", + "url": "https://docs.litellm.ai/docs/providers/pydantic_ai_agent", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true + } + }, + "stability": { + "display_name": "Stability AI (`stability`)", + "url": "https://docs.litellm.ai/docs/providers/stability", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": true, + "image_edits": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "venice": { + "display_name": "Venice.ai (`venice`)", + "url": "https://docs.litellm.ai/docs/providers/venice", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "gigachat": { + "display_name": "GigaChat (`gigachat`)", + "url": "https://docs.litellm.ai/docs/providers/gigachat", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true + } + }, + "google_pse": { + "display_name": "Google PSE (`google_pse`)", + "url": "https://docs.litellm.ai/docs/search/google_pse", + "endpoints": { + "search": true + } + }, + "milvus": { + "display_name": "Milvus (`milvus`)", + "url": "https://docs.litellm.ai/docs/providers/milvus_vector_stores", + "endpoints": { + "vector_stores_search": true + } + }, + "minimax": { + "display_name": "Minimax (`minimax`)", + "url": "https://docs.litellm.ai/docs/providers/minimax", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + }, + "pg_vector": { + "display_name": "PG Vector (`pg_vector`)", + "url": "https://docs.litellm.ai/docs/providers/pg_vector", + "endpoints": { + "vector_stores_search": true + } + }, + "helicone": { + "display_name": "Helicone (`helicone`)", + "url": "https://docs.litellm.ai/docs/providers/helicone", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + }, + "llamagate": { + "display_name": "LlamaGate (`llamagate`)", + "url": "https://docs.litellm.ai/docs/providers/llamagate", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + }, + "xiaomi_mimo": { + "display_name": "Xiaomi Mimo (`xiaomi_mimo`)", + "url": "https://docs.litellm.ai/docs/providers/xiaomi_mimo", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + }, + "manus": { + "display_name": "Manus (`manus`)", + "url": "https://docs.litellm.ai/docs/providers/manus", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "files": true, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "sarvam": { + "display_name": "Sarvam (`sarvam`)", + "url": "https://docs.litellm.ai/docs/providers/sarvam", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + } + }, + "endpoints": { + "a2a": { + "docs_label": "a2a", + "display_name": "A2A (Agent-to-Agent) protocol for agent communication", + "leftnav_label": "/a2a", + "provider_json_field": "a2a", + "url": "https://docs.litellm.ai/docs/a2a", + "bridges_to_chat_completion": true + }, + "messages": { + "docs_label": "anthropic_unified", + "display_name": "Anthropic Messages API", + "leftnav_label": "/messages", + "provider_json_field": "messages", + "url": "https://docs.litellm.ai/docs/anthropic_unified", + "bridges_to_chat_completion": true + }, + "anthropic_count_tokens": { + "docs_label": "anthropic_count_tokens", + "display_name": "Anthropic Count Tokens API", + "leftnav_label": "/count_tokens", + "provider_json_field": "count_tokens", + "url": "https://docs.litellm.ai/docs/anthropic_count_tokens" + }, + "apply_guardrail": { + "docs_label": "apply_guardrail", + "display_name": "Unified Apply Guardrail API", + "leftnav_label": "/guardrails/apply_guardrail", + "provider_json_field": "apply_guardrail", + "url": "https://docs.litellm.ai/docs/apply_guardrail" + }, + "assistants": { + "docs_label": "assistants", + "display_name": "OpenAI Assistants API", + "leftnav_label": "/assistants", + "provider_json_field": "assistants", + "url": "https://docs.litellm.ai/docs/assistants" + }, + "audio_transcription": { + "docs_label": "audio_transcription", + "display_name": "OpenAI Audio Transcription API", + "leftnav_label": "/audio/transcriptions", + "provider_json_field": "audio_transcriptions", + "url": "https://docs.litellm.ai/docs/audio_transcription" + }, + "batches": { + "docs_label": "batches", + "display_name": "OpenAI Batches API", + "leftnav_label": "/batches", + "provider_json_field": "batches", + "url": "https://docs.litellm.ai/docs/batches" + }, + "bedrock_invoke": { + "docs_label": "bedrock_invoke", + "display_name": "Bedrock Invoke API", + "leftnav_label": "/invoke", + "provider_json_field": "bedrock_invoke", + "url": "https://docs.litellm.ai/docs/bedrock_invoke" + }, + "bedrock_converse": { + "docs_label": "bedrock_converse", + "display_name": "Bedrock Converse API", + "leftnav_label": "/converse", + "provider_json_field": "bedrock_converse", + "url": "https://docs.litellm.ai/docs/bedrock_converse" + }, + "chat_completions": { + "docs_label": "chat_completions", + "display_name": "OpenAI Chat Completions API", + "leftnav_label": "/chat/completions", + "provider_json_field": "chat_completions", + "url": "https://docs.litellm.ai/docs/chat_completions" + }, + "container_files": { + "docs_label": "container_files", + "display_name": "OpenAI Container Files API", + "leftnav_label": "/create/container/files", + "provider_json_field": "container_files", + "url": "https://docs.litellm.ai/docs/container_files" + }, + "container": { + "docs_label": "containers", + "display_name": "OpenAI Containers API", + "leftnav_label": "/container", + "provider_json_field": "container", + "url": "https://docs.litellm.ai/docs/containers" + }, + "embeddings": { + "docs_label": "embedding/supported_embedding", + "display_name": "OpenAI Embeddings API", + "leftnav_label": "/embeddings", + "provider_json_field": "embeddings", + "url": "https://docs.litellm.ai/docs/embedding/supported_embedding" + }, + "files": { + "docs_label": "files", + "display_name": "OpenAI Files API", + "leftnav_label": "/files", + "provider_json_field": "files", + "url": "https://docs.litellm.ai/docs/proxy/litellm_managed_files" + }, + "fine_tuning": { + "docs_label": "fine_tuning", + "display_name": "OpenAI Fine-Tuning API", + "leftnav_label": "/fine_tuning", + "provider_json_field": "fine_tuning", + "url": "https://docs.litellm.ai/docs/proxy/managed_finetuning" + }, + "generateContent": { + "docs_label": "generateContent", + "display_name": "Google GenerateContent API", + "leftnav_label": "/generateContent", + "provider_json_field": "generateContent", + "url": "https://docs.litellm.ai/docs/generateContent", + "bridges_to_chat_completion": true + }, + "image_edits": { + "docs_label": "image_edits", + "display_name": "OpenAI Images Edits API", + "leftnav_label": "/images/edits", + "provider_json_field": "image_edits", + "url": "https://docs.litellm.ai/docs/image_edits" + }, + "image_generations": { + "docs_label": "image_generation", + "display_name": "OpenAI Images Generations API", + "leftnav_label": "/images/generations", + "provider_json_field": "image_generations", + "url": "https://docs.litellm.ai/docs/image_generation" + }, + "image_variations": { + "docs_label": "image_variations", + "display_name": "OpenAI Images Variations API", + "leftnav_label": "/images/variations", + "provider_json_field": "image_variations", + "url": "https://docs.litellm.ai/docs/image_variations" + }, + "interactions": { + "docs_label": "interactions", + "display_name": "Google Interactions API", + "leftnav_label": "/interactions", + "provider_json_field": "interactions", + "url": "https://docs.litellm.ai/docs/interactions", + "bridges_to_chat_completion": true + }, + "mcp": { + "docs_label": "mcp", + "display_name": "Model Context Protocol (MCP)", + "leftnav_label": "/mcp", + "provider_json_field": "mcp", + "url": "https://docs.litellm.ai/docs/mcp" + }, + "moderation": { + "docs_label": "moderation", + "display_name": "OpenAI Moderations API", + "leftnav_label": "/moderations", + "provider_json_field": "moderations", + "url": "https://docs.litellm.ai/docs/moderation" + }, + "ocr": { + "docs_label": "ocr", + "display_name": "Mistral OCR API", + "leftnav_label": "/ocr", + "provider_json_field": "ocr", + "url": "https://docs.litellm.ai/docs/ocr" + }, + "rag_ingest": { + "docs_label": "rag_ingest", + "display_name": "RAG Ingest API", + "leftnav_label": "/rag/ingest", + "provider_json_field": "rag_ingest", + "url": "https://docs.litellm.ai/docs/rag_ingest" + }, + "rag_query": { + "docs_label": "rag_query", + "display_name": "RAG Query API", + "leftnav_label": "/rag/query", + "provider_json_field": "rag_query", + "url": "https://docs.litellm.ai/docs/rag_query" + }, + "realtime": { + "docs_label": "realtime", + "display_name": "OpenAI Realtime API", + "leftnav_label": "/realtime", + "provider_json_field": "realtime", + "url": "https://docs.litellm.ai/docs/realtime" + }, + "rerank": { + "docs_label": "rerank", + "display_name": "Cohere Rerank API", + "leftnav_label": "/rerank", + "provider_json_field": "rerank", + "url": "https://docs.litellm.ai/docs/rerank" + }, + "responses": { + "docs_label": "response_api", + "display_name": "OpenAI Responses API", + "leftnav_label": "/responses", + "provider_json_field": "responses", + "url": "https://docs.litellm.ai/docs/response_api", + "bridges_to_chat_completion": true + }, + "response_api_compact": { + "docs_label": "response_api_compact", + "display_name": "OpenAI Responses API", + "leftnav_label": "/responses", + "provider_json_field": "compact", + "url": "https://docs.litellm.ai/docs/response_api" + }, + "search": { + "docs_label": "search", + "display_name": "Search API", + "leftnav_label": "/search", + "provider_json_field": "search", + "url": "https://docs.litellm.ai/docs/search" + }, + "skills": { + "docs_label": "skills", + "display_name": "Anthropic Skills API", + "leftnav_label": "/skills", + "provider_json_field": "skills", + "url": "https://docs.litellm.ai/docs/skills" + }, + "text_completion": { + "docs_label": "text_completion", + "display_name": "OpenAI Completions API", + "leftnav_label": "/completions", + "provider_json_field": "text_completion", + "url": "https://docs.litellm.ai/docs/text_completion", + "bridges_to_chat_completion": true + }, + "text_to_speech": { + "docs_label": "text_to_speech", + "display_name": "OpenAI Text-to-Speech API", + "leftnav_label": "/audio/speech", + "provider_json_field": "audio_speech", + "url": "https://docs.litellm.ai/docs/text_to_speech" + }, + "vector_store_files": { + "docs_label": "vector_store_files", + "display_name": "OpenAI Vector Store Files API", + "leftnav_label": "/vector_stores/files", + "provider_json_field": "vector_store_files", + "url": "https://docs.litellm.ai/docs/vector_store_files" + }, + "vector_stores_create": { + "docs_label": "vector_stores_create", + "display_name": "OpenAI Vector Stores Create API", + "leftnav_label": "/vector_stores/create", + "provider_json_field": "vector_stores_create", + "url": "https://docs.litellm.ai/docs/vector_stores/create" + }, + "vector_stores_search": { + "docs_label": "vector_stores_search", + "display_name": "OpenAI Vector Stores Search API", + "leftnav_label": "/vector_stores/search", + "provider_json_field": "vector_stores_search", + "url": "https://docs.litellm.ai/docs/vector_stores/search" + }, + "videos": { + "docs_label": "videos", + "display_name": "OpenAI Videos API", + "leftnav_label": "/videos", + "provider_json_field": "video_generations", + "url": "https://docs.litellm.ai/docs/videos" + } + } +} diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 29c9cb571ca..ff90c778d8a 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -1,6 +1,8 @@ import json import os -from typing import List +import re +from importlib.resources import files +from typing import Any, Dict, List, Optional import litellm from fastapi import APIRouter, Depends, HTTPException @@ -23,11 +25,95 @@ from litellm.types.proxy.public_endpoints.public_endpoints import ( AgentCreateInfo, ProviderCreateInfo, PublicModelHubInfo, + SupportedEndpointsResponse, ) from litellm.types.utils import LlmProviders router = APIRouter() +# --------------------------------------------------------------------------- +# /public/endpoints — helpers +# --------------------------------------------------------------------------- + +_ENDPOINT_METADATA: Dict[str, Dict[str, str]] = { + "chat_completions": {"label": "Chat Completions", "endpoint": "/chat/completions"}, + "messages": {"label": "Messages", "endpoint": "/messages"}, + "responses": {"label": "Responses", "endpoint": "/responses"}, + "embeddings": {"label": "Embeddings", "endpoint": "/embeddings"}, + "image_generations": {"label": "Image Generations", "endpoint": "/images/generations"}, + "audio_transcriptions": {"label": "Audio Transcriptions", "endpoint": "/audio/transcriptions"}, + "audio_speech": {"label": "Audio Speech", "endpoint": "/audio/speech"}, + "moderations": {"label": "Moderations", "endpoint": "/moderations"}, + "batches": {"label": "Batches", "endpoint": "/batches"}, + "rerank": {"label": "Rerank", "endpoint": "/rerank"}, + "ocr": {"label": "OCR", "endpoint": "/ocr"}, + "search": {"label": "Search", "endpoint": "/search"}, + "skills": {"label": "Skills", "endpoint": "/skills"}, + "interactions": {"label": "Interactions", "endpoint": "/interactions"}, + "a2a_(Agent Gateway)": {"label": "A2A (Agent Gateway)", "endpoint": "/a2a/{agent}/message/send"}, + "container": {"label": "Containers", "endpoint": "/containers"}, + "container_file": {"label": "Container Files", "endpoint": "/containers/{id}/files"}, + "compact": {"label": "Compact", "endpoint": "/responses/compact"}, + "files": {"label": "Files", "endpoint": "/files"}, + "image_edits": {"label": "Image Edits", "endpoint": "/images/edits"}, + "vector_stores_create": {"label": "Vector Stores (Create)", "endpoint": "/vector_stores"}, + "vector_stores_search": {"label": "Vector Stores (Search)", "endpoint": "/vector_stores/{id}/search"}, + "video_generations": {"label": "Video Generations", "endpoint": "/videos/generations"}, +} + +_SLUG_SUFFIX_RE = re.compile(r"\s*\(`[^`]+`\)\s*$") + +# Loaded once on first request; never invalidated (local file, no TTL needed). +_cached_endpoints: Optional[List[Dict[str, Any]]] = None + + +def _clean_display_name(raw: str) -> str: + return _SLUG_SUFFIX_RE.sub("", raw).strip() + + +def _build_endpoints(raw: Dict[str, Any]) -> List[Dict[str, Any]]: + """Transform raw provider_endpoints_support_backup.json into the response shape.""" + providers: Dict[str, Any] = raw.get("providers", {}) + + # Collect endpoint keys in insertion order (union across all providers). + seen: set = set() + all_keys: List[str] = [] + for provider_data in providers.values(): + for key in provider_data.get("endpoints", {}): + if key not in seen: + seen.add(key) + all_keys.append(key) + + result: List[Dict[str, Any]] = [] + for key in all_keys: + meta = _ENDPOINT_METADATA.get(key) + label = meta["label"] if meta else key.replace("_", " ").title() + path = meta["endpoint"] if meta else "/" + key.replace("_", "/") + + supporting: List[Dict[str, str]] = [ + { + "slug": slug, + "display_name": _clean_display_name(pd.get("display_name", slug)), + } + for slug, pd in providers.items() + if pd.get("endpoints", {}).get(key) + ] + result.append({"key": key, "label": label, "endpoint": path, "providers": supporting}) + + return result + + +def _load_endpoints() -> List[Dict[str, Any]]: + raw = json.loads( + files("litellm") + .joinpath("provider_endpoints_support_backup.json") + .read_text(encoding="utf-8") + ) + return _build_endpoints(raw) + + +# --------------------------------------------------------------------------- + @router.get( "/public/model_hub", @@ -225,6 +311,24 @@ async def get_litellm_blog_posts(): return BlogPostsResponse(posts=posts) +@router.get( + "/public/endpoints", + tags=["public"], + response_model=SupportedEndpointsResponse, +) +async def get_supported_endpoints() -> SupportedEndpointsResponse: + """ + Return the list of LiteLLM proxy endpoints and which providers support each one. + + Reads from the bundled local backup file. Result is cached in-process for + the lifetime of the server process. + """ + global _cached_endpoints + if _cached_endpoints is None: + _cached_endpoints = _load_endpoints() + return SupportedEndpointsResponse(endpoints=_cached_endpoints) + + @router.get( "/public/agents/fields", tags=["public", "[beta] Agents"], @@ -233,7 +337,7 @@ async def get_litellm_blog_posts(): async def get_agent_fields() -> List[AgentCreateInfo]: """ Return agent type metadata required by the dashboard create-agent flow. - + If an agent has `inherit_credentials_from_provider`, the provider's credential fields are automatically appended to the agent's credential_fields. """ @@ -242,19 +346,19 @@ async def get_agent_fields() -> List[AgentCreateInfo]: "proxy", "public_endpoints", ) - + agent_create_fields_path = os.path.join(base_path, "agent_create_fields.json") provider_create_fields_path = os.path.join(base_path, "provider_create_fields.json") with open(agent_create_fields_path, "r") as f: agent_create_fields = json.load(f) - + with open(provider_create_fields_path, "r") as f: provider_create_fields = json.load(f) - + # Build a lookup map for providers by name provider_map = {p["provider"]: p for p in provider_create_fields} - + # Merge inherited credential fields for agent in agent_create_fields: inherit_from = agent.get("inherit_credentials_from_provider") diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index 57d68771c7f..caa9a978530 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -52,3 +52,19 @@ class AgentCreateInfo(BaseModel): credential_fields: List[AgentCredentialField] litellm_params_template: Optional[Dict[str, str]] = None model_template: Optional[str] = None + + +class EndpointProvider(BaseModel): + slug: str + display_name: str + + +class SupportedEndpoint(BaseModel): + key: str + label: str + endpoint: str + providers: List[EndpointProvider] + + +class SupportedEndpointsResponse(BaseModel): + endpoints: List[SupportedEndpoint] diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 5f5e2cf1ff8..53c98c8c400 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -357,3 +357,164 @@ def test_public_model_hub_mixed_health_statuses(): assert claude["health_checked_at"] is None app.dependency_overrides.clear() + +# --------------------------------------------------------------------------- +# /public/endpoints +# --------------------------------------------------------------------------- + +import litellm.proxy.public_endpoints.public_endpoints as _pe_module +from litellm.proxy.public_endpoints.public_endpoints import _build_endpoints, _clean_display_name + + +@pytest.fixture(autouse=False) +def reset_endpoints_cache(): + """Reset the module-level cache before and after each cache-related test.""" + original = _pe_module._cached_endpoints + _pe_module._cached_endpoints = None + yield + _pe_module._cached_endpoints = original + + +def _make_client(): + app = FastAPI() + app.include_router(router) + return TestClient(app) + + +def test_get_supported_endpoints_returns_200(reset_endpoints_cache): + response = _make_client().get("/public/endpoints") + assert response.status_code == 200 + + +def test_get_supported_endpoints_response_shape(reset_endpoints_cache): + data = _make_client().get("/public/endpoints").json() + assert "endpoints" in data + assert isinstance(data["endpoints"], list) + assert len(data["endpoints"]) > 0 + + +def test_get_supported_endpoints_item_fields(reset_endpoints_cache): + endpoints = _make_client().get("/public/endpoints").json()["endpoints"] + for item in endpoints: + assert "key" in item + assert "label" in item + assert "endpoint" in item + assert "providers" in item + assert isinstance(item["providers"], list) + + +def test_get_supported_endpoints_provider_fields(reset_endpoints_cache): + endpoints = _make_client().get("/public/endpoints").json()["endpoints"] + for item in endpoints: + for provider in item["providers"]: + assert "slug" in provider + assert "display_name" in provider + + +def test_get_supported_endpoints_paths_start_with_slash(reset_endpoints_cache): + endpoints = _make_client().get("/public/endpoints").json()["endpoints"] + for item in endpoints: + assert item["endpoint"].startswith("/"), f"Expected path starting with /, got: {item['endpoint']}" + + +def test_get_supported_endpoints_chat_completions_present(reset_endpoints_cache): + endpoints = _make_client().get("/public/endpoints").json()["endpoints"] + keys = [item["key"] for item in endpoints] + assert "chat_completions" in keys + + chat = next(item for item in endpoints if item["key"] == "chat_completions") + assert chat["endpoint"] == "/chat/completions" + assert chat["label"] == "Chat Completions" + assert len(chat["providers"]) > 0 + + +def test_get_supported_endpoints_display_names_have_no_slug_suffix(reset_endpoints_cache): + """Provider display_names must not contain the raw `` (`slug`) `` suffix.""" + import re + suffix_re = re.compile(r"\(`[^`]+`\)") + endpoints = _make_client().get("/public/endpoints").json()["endpoints"] + for item in endpoints: + for provider in item["providers"]: + assert not suffix_re.search(provider["display_name"]), ( + f"display_name still contains slug suffix: {provider['display_name']!r}" + ) + + +def test_get_supported_endpoints_is_cached(reset_endpoints_cache): + """`_load_endpoints` is called only once; subsequent requests use the cache.""" + client = _make_client() + with patch( + "litellm.proxy.public_endpoints.public_endpoints._load_endpoints", + wraps=_pe_module._load_endpoints, + ) as mock_load: + client.get("/public/endpoints") + client.get("/public/endpoints") + client.get("/public/endpoints") + + mock_load.assert_called_once() + + +# --------------------------------------------------------------------------- +# _build_endpoints unit tests (transformation logic) +# --------------------------------------------------------------------------- + +_MINIMAL_RAW = { + "providers": { + "openai": { + "display_name": "OpenAI (`openai`)", + "url": "https://example.com", + "endpoints": {"chat_completions": True, "embeddings": True, "images": False}, + }, + "anthropic": { + "display_name": "Anthropic (`anthropic`)", + "url": "https://example.com", + "endpoints": {"chat_completions": True, "embeddings": False, "images": False}, + }, + } +} + + +def test_build_endpoints_known_key_uses_metadata(): + result = _build_endpoints(_MINIMAL_RAW) + chat = next(e for e in result if e["key"] == "chat_completions") + assert chat["label"] == "Chat Completions" + assert chat["endpoint"] == "/chat/completions" + + +def test_build_endpoints_only_includes_supporting_providers(): + result = _build_endpoints(_MINIMAL_RAW) + embeddings = next(e for e in result if e["key"] == "embeddings") + slugs = [p["slug"] for p in embeddings["providers"]] + assert slugs == ["openai"] + + +def test_build_endpoints_unknown_key_derives_label_and_path(): + raw = { + "providers": { + "someprovider": { + "display_name": "Some Provider (`someprovider`)", + "endpoints": {"my_custom_endpoint": True}, + } + } + } + result = _build_endpoints(raw) + item = result[0] + assert item["key"] == "my_custom_endpoint" + assert item["label"] == "My Custom Endpoint" + assert item["endpoint"].startswith("/") + + +def test_build_endpoints_empty_providers_returns_empty(): + result = _build_endpoints({"providers": {}}) + assert result == [] + + +def test_clean_display_name_strips_suffix(): + assert _clean_display_name("OpenAI (`openai`)") == "OpenAI" + assert _clean_display_name("AI/ML API (`aiml`)") == "AI/ML API" + assert _clean_display_name("A2A (Agent-to-Agent) (`a2a`)") == "A2A (Agent-to-Agent)" + + +def test_clean_display_name_passthrough_when_no_suffix(): + assert _clean_display_name("OpenAI") == "OpenAI" + assert _clean_display_name("") == "" From ffc00c0c90fa21ff6456de972d40daab8fbe2d32 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 18:30:28 -0800 Subject: [PATCH 24/54] fix: correct _ENDPOINT_METADATA keys to match actual JSON data (a2a, container_files) --- litellm/proxy/public_endpoints/public_endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index ff90c778d8a..247d386a96e 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -50,9 +50,9 @@ _ENDPOINT_METADATA: Dict[str, Dict[str, str]] = { "search": {"label": "Search", "endpoint": "/search"}, "skills": {"label": "Skills", "endpoint": "/skills"}, "interactions": {"label": "Interactions", "endpoint": "/interactions"}, - "a2a_(Agent Gateway)": {"label": "A2A (Agent Gateway)", "endpoint": "/a2a/{agent}/message/send"}, + "a2a": {"label": "A2A (Agent Gateway)", "endpoint": "/a2a/{agent}/message/send"}, "container": {"label": "Containers", "endpoint": "/containers"}, - "container_file": {"label": "Container Files", "endpoint": "/containers/{id}/files"}, + "container_files": {"label": "Container Files", "endpoint": "/containers/{id}/files"}, "compact": {"label": "Compact", "endpoint": "/responses/compact"}, "files": {"label": "Files", "endpoint": "/files"}, "image_edits": {"label": "Image Edits", "endpoint": "/images/edits"}, From fc69d6e8d1e9f4e9fe1754cd6f7a0daeba2afd23 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 19:18:17 -0800 Subject: [PATCH 25/54] fix: add 12 missing endpoint keys to _ENDPOINT_METADATA, fix stale _schema keys in backup JSON --- litellm/provider_endpoints_support_backup.json | 4 ++-- litellm/proxy/public_endpoints/public_endpoints.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 8834d8b19c0..fc79ba54759 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -19,9 +19,9 @@ "search": "Supports /search endpoint", "skills": "Supports /skills endpoint", "interactions": "Supports /interactions endpoint (Google AI Interactions API)", - "a2a_(Agent Gateway)": "Supports /a2a/{agent}/message/send endpoint (A2A Protocol)", + "a2a": "Supports /a2a/{agent}/message/send endpoint (A2A Protocol)", "container": "Supports OpenAI's /containers endpoint", - "container_file": "Supports OpenAI's /containers/{id}/files endpoint", + "container_files": "Supports OpenAI's /containers/{id}/files endpoint", "compact": "Supports /responses/compact endpoint", "files": "Supports /files endpoint for file operations", "image_edits": "Supports /images/edits endpoint for image editing", diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 247d386a96e..d611a454010 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -58,7 +58,19 @@ _ENDPOINT_METADATA: Dict[str, Dict[str, str]] = { "image_edits": {"label": "Image Edits", "endpoint": "/images/edits"}, "vector_stores_create": {"label": "Vector Stores (Create)", "endpoint": "/vector_stores"}, "vector_stores_search": {"label": "Vector Stores (Search)", "endpoint": "/vector_stores/{id}/search"}, + "vector_store_files": {"label": "Vector Store Files", "endpoint": "/vector_stores/{id}/files"}, "video_generations": {"label": "Video Generations", "endpoint": "/videos/generations"}, + "assistants": {"label": "Assistants", "endpoint": "/assistants"}, + "fine_tuning": {"label": "Fine Tuning", "endpoint": "/fine_tuning/jobs"}, + "text_completion": {"label": "Text Completion", "endpoint": "/completions"}, + "realtime": {"label": "Realtime", "endpoint": "/realtime"}, + "count_tokens": {"label": "Count Tokens", "endpoint": "/utils/token_counter"}, + "image_variations": {"label": "Image Variations", "endpoint": "/images/variations"}, + "generateContent": {"label": "Generate Content", "endpoint": "/generateContent"}, + "bedrock_invoke": {"label": "Bedrock Invoke", "endpoint": "/bedrock/invoke"}, + "bedrock_converse": {"label": "Bedrock Converse", "endpoint": "/bedrock/converse"}, + "rag_ingest": {"label": "RAG Ingest", "endpoint": "/rag/ingest"}, + "rag_query": {"label": "RAG Query", "endpoint": "/rag/query"}, } _SLUG_SUFFIX_RE = re.compile(r"\s*\(`[^`]+`\)\s*$") From 369c0ec3924b6e7d9d1ca4071c570bb7cf706e6b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:03:25 -0800 Subject: [PATCH 26/54] [Infra] Add prisma_schema_sync CircleCI job before e2e UI tests Adds a new CircleCI job that runs the proxy with --use_prisma_db_push against the base Neon branch before the e2e UI tests create their branches from it, ensuring the schema is synced on the parent. Co-Authored-By: Claude Sonnet 4.6 --- .circleci/config.yml | 68 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index fbbb6deeba8..8709f730c23 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4100,6 +4100,63 @@ jobs: path: playwright-report destination: playwright-report + prisma_schema_sync: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Install Neon CLI + command: | + npm i -g neonctl + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Sync schema on base e2e database + command: | + BASE_DATABASE_URL=$(neon connection-string \ + --project-id $NEON_PROJECT_ID \ + --api-key $NEON_API_KEY \ + --branch br-fancy-paper-ad1olsb3 \ + --database-name yuneng-trial-db \ + --role neondb_owner) + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=$BASE_DATABASE_URL \ + -e LITELLM_MASTER_KEY="sk-1234" \ + --name schema-sync \ + -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --use_prisma_db_push + - run: + name: Start outputting logs + command: docker logs -f schema-sync + background: true + - run: + name: Wait for proxy to be ready (schema sync complete) + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Stop schema sync container + command: docker stop schema-sync + test_nonroot_image: machine: image: ubuntu-2204:2023.10.1 @@ -4298,6 +4355,15 @@ workflows: only: - main - /litellm_.*/ + - prisma_schema_sync: + context: e2e_ui_tests + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ - e2e_ui_testing: name: e2e_ui_testing_chromium browser: chromium @@ -4305,6 +4371,7 @@ workflows: requires: - ui_build - build_docker_database_image + - prisma_schema_sync filters: branches: only: @@ -4317,6 +4384,7 @@ workflows: requires: - ui_build - build_docker_database_image + - prisma_schema_sync filters: branches: only: From 516b18fecaa49a39d08217d30c3f59faf149e871 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:07:51 -0800 Subject: [PATCH 27/54] [Feature] Access group CRUD: Add bidirectional sync for teams/keys When creating, updating, or deleting access groups, automatically keep team and key access_group_ids in sync with the access group's assigned_team_ids and assigned_key_ids. Includes transaction-based DB updates, cache patching, and handles out-of-sync data by unioning assigned_* fields with hasSome queries. Adds 12 new tests covering sync behavior across all three CRUD operations. Co-Authored-By: Claude Sonnet 4.6 --- .../access_group_endpoints.py | 363 +++++++++++++----- .../test_access_group_endpoints.py | 284 ++++++++++++++ 2 files changed, 560 insertions(+), 87 deletions(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 12aa748bbc3..c6958240d9a 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Set from fastapi import APIRouter, Depends, HTTPException, status @@ -94,6 +94,183 @@ async def _invalidate_cache_access_group(access_group_id: str) -> None: ) +# --------------------------------------------------------------------------- +# DB sync helpers (called inside a Prisma transaction) +# --------------------------------------------------------------------------- + + +async def _sync_add_access_group_to_teams( + tx, team_ids: List[str], access_group_id: str +) -> None: + """Add access_group_id to each team's access_group_ids (idempotent).""" + for team_id in team_ids: + team = await tx.litellm_teamtable.find_unique(where={"team_id": team_id}) + if team is not None and access_group_id not in (team.access_group_ids or []): + await tx.litellm_teamtable.update( + where={"team_id": team_id}, + data={"access_group_ids": list(team.access_group_ids or []) + [access_group_id]}, + ) + + +async def _sync_remove_access_group_from_teams( + tx, team_ids: List[str], access_group_id: str +) -> None: + """Remove access_group_id from each team's access_group_ids (idempotent).""" + for team_id in team_ids: + team = await tx.litellm_teamtable.find_unique(where={"team_id": team_id}) + if team is not None and access_group_id in (team.access_group_ids or []): + await tx.litellm_teamtable.update( + where={"team_id": team_id}, + data={"access_group_ids": [ag for ag in team.access_group_ids if ag != access_group_id]}, + ) + + +async def _sync_add_access_group_to_keys( + tx, key_tokens: List[str], access_group_id: str +) -> None: + """Add access_group_id to each key's access_group_ids (idempotent).""" + for token in key_tokens: + key = await tx.litellm_verificationtoken.find_unique(where={"token": token}) + if key is not None and access_group_id not in (key.access_group_ids or []): + await tx.litellm_verificationtoken.update( + where={"token": token}, + data={"access_group_ids": list(key.access_group_ids or []) + [access_group_id]}, + ) + + +async def _sync_remove_access_group_from_keys( + tx, key_tokens: List[str], access_group_id: str +) -> None: + """Remove access_group_id from each key's access_group_ids (idempotent).""" + for token in key_tokens: + key = await tx.litellm_verificationtoken.find_unique(where={"token": token}) + if key is not None and access_group_id in (key.access_group_ids or []): + await tx.litellm_verificationtoken.update( + where={"token": token}, + data={"access_group_ids": [ag for ag in key.access_group_ids if ag != access_group_id]}, + ) + + +# --------------------------------------------------------------------------- +# Cache patch helpers +# --------------------------------------------------------------------------- + + +async def _patch_team_caches_add_access_group( + team_ids: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached team objects to include access_group_id.""" + for team_id in team_ids: + cached_team = await _get_team_object_from_cache( + key="team_id:{}".format(team_id), + proxy_logging_obj=proxy_logging_obj, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + ) + if cached_team is None: + continue + if cached_team.access_group_ids is None: + cached_team.access_group_ids = [access_group_id] + elif access_group_id not in cached_team.access_group_ids: + cached_team.access_group_ids = list(cached_team.access_group_ids) + [access_group_id] + else: + continue + await _cache_team_object( + team_id=team_id, + team_table=cached_team, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +async def _patch_team_caches_remove_access_group( + team_ids: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached team objects to remove access_group_id.""" + for team_id in team_ids: + cached_team = await _get_team_object_from_cache( + key="team_id:{}".format(team_id), + proxy_logging_obj=proxy_logging_obj, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + ) + if cached_team is not None and cached_team.access_group_ids: + cached_team.access_group_ids = [ + ag for ag in cached_team.access_group_ids if ag != access_group_id + ] + await _cache_team_object( + team_id=team_id, + team_table=cached_team, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +async def _patch_key_caches_add_access_group( + key_tokens: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached key objects to include access_group_id.""" + for token in key_tokens: + cached_key = await user_api_key_cache.async_get_cache(key=token) + if cached_key is None: + continue + if isinstance(cached_key, dict): + cached_key = UserAPIKeyAuth(**cached_key) + if not isinstance(cached_key, UserAPIKeyAuth): + continue + if cached_key.access_group_ids is None: + cached_key.access_group_ids = [access_group_id] + elif access_group_id not in cached_key.access_group_ids: + cached_key.access_group_ids = list(cached_key.access_group_ids) + [access_group_id] + else: + continue + await _cache_key_object( + hashed_token=token, + user_api_key_obj=cached_key, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +async def _patch_key_caches_remove_access_group( + key_tokens: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached key objects to remove access_group_id.""" + for token in key_tokens: + cached_key = await user_api_key_cache.async_get_cache(key=token) + if cached_key is None: + continue + if isinstance(cached_key, dict): + cached_key = UserAPIKeyAuth(**cached_key) + if isinstance(cached_key, UserAPIKeyAuth) and cached_key.access_group_ids: + cached_key.access_group_ids = [ + ag for ag in cached_key.access_group_ids if ag != access_group_id + ] + await _cache_key_object( + hashed_token=token, + user_api_key_obj=cached_key, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +# --------------------------------------------------------------------------- +# CRUD endpoints +# --------------------------------------------------------------------------- + + @router.post( "/v1/access_group", response_model=AccessGroupResponse, @@ -106,32 +283,42 @@ async def create_access_group( _require_proxy_admin(user_api_key_dict) prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - existing = await prisma_client.db.litellm_accessgrouptable.find_unique( - where={"access_group_name": data.access_group_name} - ) - if existing is not None: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=f"Access group '{data.access_group_name}' already exists", - ) - try: - record = await prisma_client.db.litellm_accessgrouptable.create( - data={ - "access_group_name": data.access_group_name, - "description": data.description, - "access_model_names": data.access_model_names or [], - "access_mcp_server_ids": data.access_mcp_server_ids or [], - "access_agent_ids": data.access_agent_ids or [], - "assigned_team_ids": data.assigned_team_ids or [], - "assigned_key_ids": data.assigned_key_ids or [], - "created_by": user_api_key_dict.user_id, - "updated_by": user_api_key_dict.user_id, - } - ) + async with prisma_client.db.tx() as tx: + existing = await tx.litellm_accessgrouptable.find_unique( + where={"access_group_name": data.access_group_name} + ) + if existing is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Access group '{data.access_group_name}' already exists", + ) + + record = await tx.litellm_accessgrouptable.create( + data={ + "access_group_name": data.access_group_name, + "description": data.description, + "access_model_names": data.access_model_names or [], + "access_mcp_server_ids": data.access_mcp_server_ids or [], + "access_agent_ids": data.access_agent_ids or [], + "assigned_team_ids": data.assigned_team_ids or [], + "assigned_key_ids": data.assigned_key_ids or [], + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, + } + ) + + # Sync team and key tables to reference the new access group + await _sync_add_access_group_to_teams( + tx, data.assigned_team_ids or [], record.access_group_id + ) + await _sync_add_access_group_to_keys( + tx, data.assigned_key_ids or [], record.access_group_id + ) + except HTTPException: + raise except Exception as e: # Race condition: another request created the same name between find_unique and create. - # Prisma raises UniqueViolationError (P2002) or similar for unique constraint. if "unique constraint" in str(e).lower() or "P2002" in str(e): raise HTTPException( status_code=status.HTTP_409_CONFLICT, @@ -139,8 +326,15 @@ async def create_access_group( ) raise - # Cache the newly created access group for read-heavy access patterns + from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache + await _cache_access_group_record(record) + await _patch_team_caches_add_access_group( + data.assigned_team_ids or [], record.access_group_id, user_api_key_cache, proxy_logging_obj + ) + await _patch_key_caches_add_access_group( + data.assigned_key_ids or [], record.access_group_id, user_api_key_cache, proxy_logging_obj + ) return _record_to_response(record) @@ -204,15 +398,35 @@ async def update_access_group( detail=f"Access group '{access_group_id}' not found", ) + # Compute team/key assignment deltas before the transaction + update_fields = data.model_dump(exclude_unset=True) + old_team_ids: Set[str] = set(existing.assigned_team_ids or []) + old_key_ids: Set[str] = set(existing.assigned_key_ids or []) + new_team_ids: Set[str] = set(update_fields["assigned_team_ids"]) if "assigned_team_ids" in update_fields else old_team_ids + new_key_ids: Set[str] = set(update_fields["assigned_key_ids"]) if "assigned_key_ids" in update_fields else old_key_ids + + teams_to_add = list(new_team_ids - old_team_ids) + teams_to_remove = list(old_team_ids - new_team_ids) + keys_to_add = list(new_key_ids - old_key_ids) + keys_to_remove = list(old_key_ids - new_key_ids) + update_data: dict = {"updated_by": user_api_key_dict.user_id} - for field, value in data.model_dump(exclude_unset=True).items(): + for field, value in update_fields.items(): update_data[field] = value try: - record = await prisma_client.db.litellm_accessgrouptable.update( - where={"access_group_id": access_group_id}, - data=update_data, - ) + async with prisma_client.db.tx() as tx: + record = await tx.litellm_accessgrouptable.update( + where={"access_group_id": access_group_id}, + data=update_data, + ) + + await _sync_add_access_group_to_teams(tx, teams_to_add, access_group_id) + await _sync_remove_access_group_from_teams(tx, teams_to_remove, access_group_id) + await _sync_add_access_group_to_keys(tx, keys_to_add, access_group_id) + await _sync_remove_access_group_from_keys(tx, keys_to_remove, access_group_id) + except HTTPException: + raise except Exception as e: # Unique constraint violation (e.g. access_group_name already exists). if "unique constraint" in str(e).lower() or "P2002" in str(e): @@ -222,8 +436,13 @@ async def update_access_group( ) raise - # Write the updated record into cache (same key, overwrites stale entry) + from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache + await _cache_access_group_record(record) + await _patch_team_caches_add_access_group(teams_to_add, access_group_id, user_api_key_cache, proxy_logging_obj) + await _patch_team_caches_remove_access_group(teams_to_remove, access_group_id, user_api_key_cache, proxy_logging_obj) + await _patch_key_caches_add_access_group(keys_to_add, access_group_id, user_api_key_cache, proxy_logging_obj) + await _patch_key_caches_remove_access_group(keys_to_remove, access_group_id, user_api_key_cache, proxy_logging_obj) return _record_to_response(record) @@ -240,9 +459,8 @@ async def delete_access_group( prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) try: - # Track affected team IDs and key tokens for cache invalidation - affected_team_ids: list = [] - affected_key_tokens: list = [] + affected_team_ids: List[str] = [] + affected_key_tokens: List[str] = [] async with prisma_client.db.tx() as tx: existing = await tx.litellm_accessgrouptable.find_unique( @@ -254,73 +472,44 @@ async def delete_access_group( detail=f"Access group '{access_group_id}' not found", ) - # Remove access_group_id from teams and keys that reference it + # Union of: teams that have this access_group_id in their own access_group_ids + # AND teams listed in assigned_team_ids (handles out-of-sync data from before this sync was added) teams_with_group = await tx.litellm_teamtable.find_many( where={"access_group_ids": {"hasSome": [access_group_id]}} ) - for team in teams_with_group: - affected_team_ids.append(team.team_id) - updated_ids = [tid for tid in (team.access_group_ids or []) if tid != access_group_id] - await tx.litellm_teamtable.update( - where={"team_id": team.team_id}, - data={"access_group_ids": updated_ids}, - ) + all_affected_team_ids: Set[str] = ( + {team.team_id for team in teams_with_group} + | set(existing.assigned_team_ids or []) + ) + affected_team_ids = list(all_affected_team_ids) + # Union of: keys that have this access_group_id in their own access_group_ids + # AND keys listed in assigned_key_ids (handles out-of-sync data) keys_with_group = await tx.litellm_verificationtoken.find_many( where={"access_group_ids": {"hasSome": [access_group_id]}} ) - for key in keys_with_group: - affected_key_tokens.append(key.token) - updated_ids = [kid for kid in (key.access_group_ids or []) if kid != access_group_id] - await tx.litellm_verificationtoken.update( - where={"token": key.token}, - data={"access_group_ids": updated_ids}, - ) + all_affected_key_tokens: Set[str] = ( + {key.token for key in keys_with_group} + | set(existing.assigned_key_ids or []) + ) + affected_key_tokens = list(all_affected_key_tokens) + + await _sync_remove_access_group_from_teams(tx, affected_team_ids, access_group_id) + await _sync_remove_access_group_from_keys(tx, affected_key_tokens, access_group_id) await tx.litellm_accessgrouptable.delete( where={"access_group_id": access_group_id} ) - # Invalidate the deleted access group from cache - await _invalidate_cache_access_group(access_group_id) - - # Patch cached team and key objects to remove the deleted access_group_id - # instead of fully invalidating them (keeps cache warm, avoids DB re-fetch) from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache - for team_id in affected_team_ids: - cached_team = await _get_team_object_from_cache( - key="team_id:{}".format(team_id), - proxy_logging_obj=proxy_logging_obj, - user_api_key_cache=user_api_key_cache, - parent_otel_span=None, - ) - if cached_team is not None and cached_team.access_group_ids: - cached_team.access_group_ids = [ - ag_id for ag_id in cached_team.access_group_ids if ag_id != access_group_id - ] - await _cache_team_object( - team_id=team_id, - team_table=cached_team, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - - for token in affected_key_tokens: - cached_key = await user_api_key_cache.async_get_cache(key=token) - if cached_key is not None: - if isinstance(cached_key, dict): - cached_key = UserAPIKeyAuth(**cached_key) - if isinstance(cached_key, UserAPIKeyAuth) and cached_key.access_group_ids: - cached_key.access_group_ids = [ - ag_id for ag_id in cached_key.access_group_ids if ag_id != access_group_id - ] - await _cache_key_object( - hashed_token=token, - user_api_key_obj=cached_key, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + await _invalidate_cache_access_group(access_group_id) + await _patch_team_caches_remove_access_group( + affected_team_ids, access_group_id, user_api_key_cache, proxy_logging_obj + ) + await _patch_key_caches_remove_access_group( + affected_key_tokens, access_group_id, user_api_key_cache, proxy_logging_obj + ) except HTTPException: raise diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py index 9b6e0631762..a8842f7448b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -99,10 +99,12 @@ def client_and_mocks(monkeypatch): mock_team_table = MagicMock() mock_team_table.find_many = AsyncMock(return_value=[]) + mock_team_table.find_unique = AsyncMock(return_value=None) mock_team_table.update = AsyncMock(return_value=None) mock_key_table = MagicMock() mock_key_table.find_many = AsyncMock(return_value=[]) + mock_key_table.find_unique = AsyncMock(return_value=None) mock_key_table.update = AsyncMock(return_value=None) @asynccontextmanager @@ -570,11 +572,13 @@ def test_delete_access_group_cleans_up_teams_and_keys(client_and_mocks): team_with_group.team_id = "team-1" team_with_group.access_group_ids = ["ag-to-delete", "ag-other"] mock_team_table.find_many = AsyncMock(return_value=[team_with_group]) + mock_team_table.find_unique = AsyncMock(return_value=team_with_group) key_with_group = MagicMock() key_with_group.token = "key-token-1" key_with_group.access_group_ids = ["ag-to-delete"] mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + mock_key_table.find_unique = AsyncMock(return_value=key_with_group) resp = client.delete("/v1/access_group/ag-to-delete") assert resp.status_code == 204 @@ -669,11 +673,13 @@ def test_delete_access_group_patches_cached_team_and_key( team_with_group.team_id = "team-1" team_with_group.access_group_ids = ["ag-to-delete", "ag-keep"] mock_team_table.find_many = AsyncMock(return_value=[team_with_group]) + mock_team_table.find_unique = AsyncMock(return_value=team_with_group) key_with_group = MagicMock() key_with_group.token = "hashed-key-1" key_with_group.access_group_ids = ["ag-to-delete"] mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + mock_key_table.find_unique = AsyncMock(return_value=key_with_group) # Build cached team object (returned from proxy_logging dual cache) if team_cache_group_ids is not None: @@ -762,6 +768,7 @@ def test_delete_access_group_patches_key_cached_as_dict(client_and_mocks): key_with_group.token = "hashed-key-dict" key_with_group.access_group_ids = ["ag-to-delete", "ag-other"] mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + mock_key_table.find_unique = AsyncMock(return_value=key_with_group) # No team in cache mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock( @@ -882,3 +889,280 @@ def test_record_to_access_group_table(): assert result.access_group_name == "unit-test-group" assert result.access_model_names == ["gpt-4", "claude-3"] assert result.access_agent_ids == ["agent-1"] + + +# --------------------------------------------------------------------------- +# Sync tests: CREATE +# --------------------------------------------------------------------------- + + +def test_create_access_group_syncs_assigned_teams(client_and_mocks): + """Create adds access_group_id to each assigned team's access_group_ids in DB.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + team_record = MagicMock() + team_record.team_id = "team-1" + team_record.access_group_ids = [] + mock_team_table.find_unique = AsyncMock(return_value=team_record) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_team_ids": ["team-1"]}, + ) + assert resp.status_code == 201 + + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-1"}) + mock_team_table.update.assert_awaited_once() + call_kwargs = mock_team_table.update.call_args.kwargs + assert call_kwargs["where"] == {"team_id": "team-1"} + # The newly created access group id ("ag-new") should be in the updated list + assert "ag-new" in call_kwargs["data"]["access_group_ids"] + + +def test_create_access_group_syncs_assigned_keys(client_and_mocks): + """Create adds access_group_id to each assigned key's access_group_ids in DB.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + key_record = MagicMock() + key_record.token = "hashed-token-1" + key_record.access_group_ids = [] + mock_key_table.find_unique = AsyncMock(return_value=key_record) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_key_ids": ["hashed-token-1"]}, + ) + assert resp.status_code == 201 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "hashed-token-1"}) + mock_key_table.update.assert_awaited_once() + call_kwargs = mock_key_table.update.call_args.kwargs + assert call_kwargs["where"] == {"token": "hashed-token-1"} + assert "ag-new" in call_kwargs["data"]["access_group_ids"] + + +def test_create_access_group_skips_sync_for_nonexistent_team(client_and_mocks): + """Create skips updating a team that doesn't exist in DB.""" + client, mock_prisma, _, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + mock_team_table.find_unique = AsyncMock(return_value=None) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_team_ids": ["nonexistent-team"]}, + ) + assert resp.status_code == 201 + mock_team_table.update.assert_not_awaited() + + +def test_create_access_group_idempotent_team_sync(client_and_mocks): + """Create skips updating a team that already has the access_group_id.""" + client, mock_prisma, _, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + team_record = MagicMock() + team_record.team_id = "team-1" + team_record.access_group_ids = ["ag-new"] # already synced + mock_team_table.find_unique = AsyncMock(return_value=team_record) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_team_ids": ["team-1"]}, + ) + assert resp.status_code == 201 + mock_team_table.update.assert_not_awaited() + + +# --------------------------------------------------------------------------- +# Sync tests: UPDATE +# --------------------------------------------------------------------------- + + +def test_update_access_group_syncs_added_teams(client_and_mocks): + """Update adds access_group_id to newly assigned teams.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_team_ids=["team-existing"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + team_record = MagicMock() + team_record.team_id = "team-new" + team_record.access_group_ids = [] + mock_team_table.find_unique = AsyncMock(return_value=team_record) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_team_ids": ["team-existing", "team-new"]}, + ) + assert resp.status_code == 200 + + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-new"}) + mock_team_table.update.assert_awaited_once() + call_kwargs = mock_team_table.update.call_args.kwargs + assert call_kwargs["where"] == {"team_id": "team-new"} + assert "ag-update" in call_kwargs["data"]["access_group_ids"] + + +def test_update_access_group_syncs_removed_teams(client_and_mocks): + """Update removes access_group_id from de-assigned teams.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_team_ids=["team-keep", "team-remove"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + team_to_remove = MagicMock() + team_to_remove.team_id = "team-remove" + team_to_remove.access_group_ids = ["ag-update"] + mock_team_table.find_unique = AsyncMock(return_value=team_to_remove) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_team_ids": ["team-keep"]}, + ) + assert resp.status_code == 200 + + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-remove"}) + mock_team_table.update.assert_awaited_once() + call_kwargs = mock_team_table.update.call_args.kwargs + assert call_kwargs["where"] == {"team_id": "team-remove"} + assert "ag-update" not in call_kwargs["data"]["access_group_ids"] + + +def test_update_access_group_no_team_sync_when_ids_not_in_payload(client_and_mocks): + """Update does not sync teams when assigned_team_ids is absent from the payload.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_team_ids=["team-1"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + resp = client.put("/v1/access_group/ag-update", json={"description": "new desc"}) + assert resp.status_code == 200 + + mock_team_table.find_unique.assert_not_awaited() + mock_team_table.update.assert_not_awaited() + + +def test_update_access_group_syncs_added_keys(client_and_mocks): + """Update adds access_group_id to newly assigned keys.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_key_ids=["old-token"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + key_record = MagicMock() + key_record.token = "new-token" + key_record.access_group_ids = [] + mock_key_table.find_unique = AsyncMock(return_value=key_record) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_key_ids": ["old-token", "new-token"]}, + ) + assert resp.status_code == 200 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "new-token"}) + mock_key_table.update.assert_awaited_once() + call_kwargs = mock_key_table.update.call_args.kwargs + assert call_kwargs["where"] == {"token": "new-token"} + assert "ag-update" in call_kwargs["data"]["access_group_ids"] + + +def test_update_access_group_syncs_removed_keys(client_and_mocks): + """Update removes access_group_id from de-assigned keys.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_key_ids=["keep-token", "remove-token"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + key_to_remove = MagicMock() + key_to_remove.token = "remove-token" + key_to_remove.access_group_ids = ["ag-update"] + mock_key_table.find_unique = AsyncMock(return_value=key_to_remove) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_key_ids": ["keep-token"]}, + ) + assert resp.status_code == 200 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "remove-token"}) + mock_key_table.update.assert_awaited_once() + call_kwargs = mock_key_table.update.call_args.kwargs + assert call_kwargs["where"] == {"token": "remove-token"} + assert "ag-update" not in call_kwargs["data"]["access_group_ids"] + + +# --------------------------------------------------------------------------- +# Sync tests: DELETE (out-of-sync data handling) +# --------------------------------------------------------------------------- + + +def test_delete_access_group_handles_out_of_sync_assigned_teams(client_and_mocks): + """Delete includes teams from assigned_team_ids even when not found by hasSome query.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + # Access group has assigned_team_ids but the team's access_group_ids is not synced + existing = _make_access_group_record( + access_group_id="ag-to-delete", + assigned_team_ids=["team-out-of-sync"], + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + # hasSome query finds nothing (team's own access_group_ids is out of sync) + mock_team_table.find_many = AsyncMock(return_value=[]) + + out_of_sync_team = MagicMock() + out_of_sync_team.team_id = "team-out-of-sync" + out_of_sync_team.access_group_ids = [] # already clean, no update needed + mock_team_table.find_unique = AsyncMock(return_value=out_of_sync_team) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 204 + + # find_unique is called for the out-of-sync team (included via union with assigned_team_ids) + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-out-of-sync"}) + # No update needed since team's access_group_ids doesn't contain "ag-to-delete" + mock_team_table.update.assert_not_awaited() + + +def test_delete_access_group_handles_out_of_sync_assigned_keys(client_and_mocks): + """Delete includes keys from assigned_key_ids even when not found by hasSome query.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record( + access_group_id="ag-to-delete", + assigned_key_ids=["token-out-of-sync"], + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + mock_key_table.find_many = AsyncMock(return_value=[]) + + out_of_sync_key = MagicMock() + out_of_sync_key.token = "token-out-of-sync" + out_of_sync_key.access_group_ids = [] + mock_key_table.find_unique = AsyncMock(return_value=out_of_sync_key) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 204 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "token-out-of-sync"}) + mock_key_table.update.assert_not_awaited() From 2d9ba674ec0178d7e2a8bccd75872ae3c4972d6b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:14:54 -0800 Subject: [PATCH 28/54] fix: move update_access_group find_unique inside transaction Eliminates TOCTOU race where existing record was read outside the transaction, allowing a concurrent update to make delta computation stale. Delta is now computed atomically within the same transaction as the write. Co-Authored-By: Claude Sonnet 4.6 --- .../access_group_endpoints.py | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index c6958240d9a..9a6ff219d4f 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -389,33 +389,34 @@ async def update_access_group( _require_proxy_admin(user_api_key_dict) prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - existing = await prisma_client.db.litellm_accessgrouptable.find_unique( - where={"access_group_id": access_group_id} - ) - if existing is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Access group '{access_group_id}' not found", - ) - - # Compute team/key assignment deltas before the transaction update_fields = data.model_dump(exclude_unset=True) - old_team_ids: Set[str] = set(existing.assigned_team_ids or []) - old_key_ids: Set[str] = set(existing.assigned_key_ids or []) - new_team_ids: Set[str] = set(update_fields["assigned_team_ids"]) if "assigned_team_ids" in update_fields else old_team_ids - new_key_ids: Set[str] = set(update_fields["assigned_key_ids"]) if "assigned_key_ids" in update_fields else old_key_ids - - teams_to_add = list(new_team_ids - old_team_ids) - teams_to_remove = list(old_team_ids - new_team_ids) - keys_to_add = list(new_key_ids - old_key_ids) - keys_to_remove = list(old_key_ids - new_key_ids) - update_data: dict = {"updated_by": user_api_key_dict.user_id} for field, value in update_fields.items(): update_data[field] = value try: async with prisma_client.db.tx() as tx: + # Read inside the transaction so delta computation is consistent with the write, + # avoiding a TOCTOU race where a concurrent update could make deltas stale. + existing = await tx.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if existing is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + + old_team_ids: Set[str] = set(existing.assigned_team_ids or []) + old_key_ids: Set[str] = set(existing.assigned_key_ids or []) + new_team_ids: Set[str] = set(update_fields["assigned_team_ids"]) if "assigned_team_ids" in update_fields else old_team_ids + new_key_ids: Set[str] = set(update_fields["assigned_key_ids"]) if "assigned_key_ids" in update_fields else old_key_ids + + teams_to_add = list(new_team_ids - old_team_ids) + teams_to_remove = list(old_team_ids - new_team_ids) + keys_to_add = list(new_key_ids - old_key_ids) + keys_to_remove = list(old_key_ids - new_key_ids) + record = await tx.litellm_accessgrouptable.update( where={"access_group_id": access_group_id}, data=update_data, From ee7b73764cd7fe023087fa2500584f966ff019b7 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:29:43 -0800 Subject: [PATCH 29/54] =?UTF-8?q?bump:=20version=200.4.48=20=E2=86=92=200.?= =?UTF-8?q?4.49?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- litellm-proxy-extras/litellm_proxy_extras/schema.prisma | 1 + litellm-proxy-extras/pyproject.toml | 4 ++-- litellm/proxy/schema.prisma | 1 + pyproject.toml | 2 +- requirements.txt | 2 +- schema.prisma | 1 + 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 440c9c1d829..34308b29ebf 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -390,6 +390,7 @@ model LiteLLM_DeletedVerificationToken { config Json @default("{}") user_id String? team_id String? + agent_id String? project_id String? permissions Json @default("{}") max_parallel_requests Int? diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index bd57b248cdb..968536712dc 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.48" +version = "0.4.49" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.48" +version = "0.4.49" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 440c9c1d829..34308b29ebf 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -390,6 +390,7 @@ model LiteLLM_DeletedVerificationToken { config Json @default("{}") user_id String? team_id String? + agent_id String? project_id String? permissions Json @default("{}") max_parallel_requests Int? diff --git a/pyproject.toml b/pyproject.toml index 8eb433fb064..07f3ea30fc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ boto3 = { version = "1.40.76", optional = true } redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"} a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.4.48", optional = true} +litellm-proxy-extras = {version = "0.4.49", optional = true} rich = {version = "13.7.1", optional = true} litellm-enterprise = {version = "0.1.32", optional = true} diskcache = {version = "^5.6.1", optional = true} diff --git a/requirements.txt b/requirements.txt index 6cdb2f63a33..67f390cf272 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,7 +57,7 @@ grpcio>=1.75.0; python_version >= "3.14" sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.48 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.49 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env diff --git a/schema.prisma b/schema.prisma index 440c9c1d829..34308b29ebf 100644 --- a/schema.prisma +++ b/schema.prisma @@ -390,6 +390,7 @@ model LiteLLM_DeletedVerificationToken { config Json @default("{}") user_id String? team_id String? + agent_id String? project_id String? permissions Json @default("{}") max_parallel_requests Int? From 1e82ec644828e28df67576aea5368268b2eb9255 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:30:10 -0800 Subject: [PATCH 30/54] adding build --- ...litellm_proxy_extras-0.4.49-py3-none-any.whl | Bin 0 -> 65579 bytes .../dist/litellm_proxy_extras-0.4.49.tar.gz | Bin 0 -> 28710 bytes .../migration.sql | 3 +++ 3 files changed, 3 insertions(+) create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49-py3-none-any.whl create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49.tar.gz create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260226202727_add_agent_id_to_delete_keys/migration.sql diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..e44b58f8e63bac05327c43d004bfbf3c2eec533d GIT binary patch literal 65579 zcmcG$1yq%5*ET90A}t`T(z$35kw&^ZB^KQwEiFh$cS(1rq;z+eAf+JP2*`Psy7%|N zyWjJ(&p!@h+{>Xu##nRSGp>2fId54B7+CCk_wFGA=dmd8aSs~u0sN2x$K1poY;JC$ zV{L2YW7N?xu{5#Q(P6N5hPfwup2XOm|6Ls@Z=NWZlIb_;&@X;%VYH491TLVF0B02*ENK_^s7 z&6?&4X;N~FJVT?DZy}wfy6OlcVg{i@?~}t_E)rZ}Lq?YmiJD!mh^Y$iqbGh8d#lJ! zCF&ddzD+bcC0YEh}L05=EMoulbAY{RDpr z7{Rc~Qi4iHQxU$;DaT7Q&;(%m^z5pm<2d-I+SrwJwD|LLRV6(J%%1W;V(w)LReW$A zwGrNV$;TSmT+%H~g_cK+=Ss0<;_GTXSQqHlE`M%|3hE3x?f*8}RRew4`|WVUDfhIi z3#$lm`w0X6$MWXkkK{9!`(pEts8Dg-QBdI$Q9=+en7WL2%88R28z!vz9e9M<_Oybf zNRSAx86*onosUbu$MMYZ{5+8=pOUq|+B~7@X^r;f$I9e&N&mAi2G@D6rwWf6^;+A2i1ZOI0$3|}iC%Y=pWVM{HUSdwh12l& z?!9EacTf1gnVp4+k*%)1iIt@tBQuDZ4aCCC&cx2l#KfYbr)vi`H?agW{{3GV>}<@} z)EBJg*wH>hl`=G+%;35`G4j z&s9?KffX^b^n7?Pf9qpPydk6eQ%v!_nbJ$-r(pvPHrZ3zc4ltnmLYO|j|Zg(-*9t% z9NwoB!T1#S1U5e|@_Kn~BPWeEmTT-m5tR zZ<5sF3e^=U4c}Bu&I7qBe!*%NJP|G$AsVw7x}O|xw{q1@yVSQ~BsnPw&@-CtNLNDw zG@goU*b0#fkw%XluVIQpjUj!*uUB5H_xR~PRqfWZ-Zw=vMg~G{Cw$6q^fUG}2_YKm zVx~!{3zLv}cm;F#_woa9fPw-YplLdVWiOX-!1|HZefshYH!WlaWOCb$*9@{E_V&P5VzC zgJrG3uhn4jO!bN+G&CIkq0lfdrvj=kZ0iP=8U!yNxj8h5>X!s2LqXL_KK=q@-Pc3~ z10unO3A;br8?LZrV`c(t20eNN70IXCaI&NBmAw*Iqto=}?31(mAn3l9rC%&keG+c) zJ!EIYQDZ_x%n}CM8OuGGATp9O$KgX8$@mzDq5~Lj(RUAB1>4`h>)k8N5W`PVVnsn5 z;(g3-wu>&b75G+zZN>^ZS4zep&?Zz|1dsD9nM!osg%ZVaQ zr&7e>YS?V3JV|0p+47M|TRlY=z%-qQ?{(d06>CNgsr4F>2Cr z;g1$z|8N?KPJUjwHjl?TRS#kte{5I2c(_8~UGpxmeK8xm0wt=759#N7Xi>$?_z4xd zl7h|(ko-<&Bp*1MFU4=CTqt+iRC@RoDffQ#xMz@|B;NXxgaWQQV3?ZOxLZM7wqe6x0r?v{*iXIG&c@+MA>opz zbToMKQMLnI#;zu}8OBE0GC^B@LinhDeBe=idg}>VZbIRPoz@Y>u57xB&$fBSPPnvR zf~iqZ{>=8^vkkY@_e%B0yo$8yHG&7k5#mPp`7{xO1;(=7(Np@FMw%ItE4u;fSQgle zb}H09f{(I@_gFH=EJ;;X^X<%LLS<|G9SBAO+ZLDZ6Lh4>Ab2`Skt~^Z&cg>VhwXCA zRwQoyBnseQb|$aVklNfJCiX0O>zt6fHvW~q;Rn3C!=_pb8%3!d@k&%H&(5@NllWZ1 z+r&;Nw?dSfB&lmv^{q>$vKPBfLL<(x^55P+j{7mvC+gJ4Y>zeIF091x&ODL((}}*P zS8#%l^Y_pEs}uIjBJ*&530nR#3Nlk{^oiHqb|!;88Oj^=2%(#-Wr@P>>v;LB!tA+~ zQ!VatiGhmqy+nGw^ryz~Ef4P#RlehG$-x?XJ=5~>I^3Tt-uMMVZe(fAV2wMQW8p?u z{24{Rbs=p~+UT@;Hwn(cxN~O>eYU(me;f-@{Y=i<0NL2Vv}W+YyDWO$?gPLD1`0&34)W<_HGO17L67s2wny{Q zo*Tb-9u~~Y$gUHL`o1-%egP^vIJsiG;N%Mr*i;=szDzyYPd~EI#IXCpNXaiB=y#T!{eUlKVHQlYe%b*ncqsI$qE`~J|H~z zQx(`$x@o5Am6RmIr*x*cPb+0w=ygRKVXHE(PJ=o*J1ZlRq0TM8)+bdgKsRHkQ5v6U zBlb}aXQ0dOA_=})X~4qsr|Ujgj;Kx-%gn8PKzKiQXmEnj&lNdcwGmriWln}Bf}ZPR zYr?4kv(ZbNc*htv+>VF17v`Ih<4_q_E90*ndIHl8%JZ6S84`5ffa$4qzxLIOZzSRz=Ub5^#k5LB(&w8fneZg7S5V?z>_Ql`qn9 zYw}^~dE!cr@27xr>fSA#gYOSD%@B`PsM6XM1nH6!dcKfydkcgTexiMTFNgP{bN0FV zg55{p~5Z8@k<^!1ytl)xmH;Y7{f>lT#*)9fViv8q$ zKiR`9j-OUzzYXoLAOFY!Au_j8QN?14NNM+{@jP4L>#q*8XnPNCf*}yOxq%6Ala|2I zz59)FFmZyIm_fg8RM*->#};hk0JgKY)3LYuKiu%c{xalN5IX!4kIa5#<8b4ci5^BV z^*1mBh7}@gp4hXqlg-n6iy%+&DVhzrarB^O{%`HXF?K5JU_0tq8FTgB`lb||Rl-5q zF7lg3~!uZX6LFj(F=MM!LXujojrHtC+m`4`2myJgn59n&t z_u`{noeos*JfY)8kJJ5p;S^5?<7`{x=%m>cUrB}NwF&Lj9IUmW(cDicgk((fz=n)$ z%?d9Iea&z?@?~fF{kOB0)++CvaALDhwNCB@vbE2#kZrm3>)cePRev{;GoZwB;6M!L z9}Bm?;qhMxnuUp(iH((wT?eS^^v!ha9Q5_Uc6K_tmIgY8Ku`c1{Erp=4idUXM*pPi zklh90;c(b8N~EZuaxuRV4z`$@e2OH7~kpYXvi z4_=v23yhyws*ZB!uU;rj_CP;F$a$|vuoRTc!KjeBrX(DsUJHK`yN&hiE&97p=PV+6 z)4Tmvg${(m4#gWKEiG|qH`Zs)fy{lJ8PqX_hrCvH8TZ@{;8-&FE&y_w3qvODf`OTsSck34FM z{iHAhV~z1a9sgpm5%OTWb8^>xo}GvdTcu6+YY z)e0P&yGzH$#Ky$V%JIvL^mX-(!GQDV8t6FM0vrkugw4Q~c7Fn~oi!NnFu><5?R52j zSF|@o59u#MZ6!s&H=JBj3z}%m)&bT1@SPY99$JF_jWzw4v< zV%nowJ%$Q9`PiG?Xf5njGgfoP_M>ZQQ~|eJ_~Lm^z>qf(p_$?S1-@VfRKdl>#mxRo z8NWlvuS@?Ye4z-z7uaup;|nhIe~(=@j3CUx7}mMOQp~-f zSW0_Bg&Du#Oa*C*vQ8ek_+c2{5bg@^{Uh(7CKX|Ye9zCs992}z3E7Y63zk`Q8-A?C zb{1kQVk4wXJ>w(x8bVJk@L_W+7x4F-kd=9xqe&+Gf;UGQM&m-4d-%aycI2?#5VHBA zH^FJkBo9hD=zJ;OJ8SJG9wihuy2%X|_XKdqfbfyHgjQQURd+gTbuI@=1%uOGe&fn7 z8IIbrg;V$rZLxSCvD)CJ0!k*hEAoLzv7h4-+>Yl+xlI~9F{=faEdE+B34-`js|Pia zaC3uX;LvAbp)4J)g5jsi!P+=1ui&8;0>^1h;731v)Ci9{`Pn6RkSv14bA&%g^y*Z# zjYtW>sHorpvss;Cb8!Fegg$CeC!r3>HeA!K3tu5Nljs00ycXbNWbbbM>`Y8-EPvpK zz3!hZ@e6zC-K1r*6s4>(K7_Cal8}Nrz9p^4|F@g%85!Nn2LuV z<+Nq*M(a{53Fol(iC!XyWnnJj-x5KR=t5ur5gX2Z0>3qDNB(*YD$gYQX?o8&R)Zu$ zpSpm}lXgdo1rB!I=R-fFC&(ECBRDLscw>i&+mCf;`dtl!a>nk19wg}4IrE~@l;9yL zEX+`x+dMn6jznjihPPpr+uL4mJFM9&@s}RZ`&NNcw^PQ*kmoWk2zwBLdMu=gi2s5Vy2u*M2WPdMj6My_{a@k;MjrMWH}t}ppoGBOxh09BW%d-+%4e~-Fh^N zk9P&Ejx0#M&G~(!zh;kq(JEaKTSf`T0;^%kfGtVFWJ8(tOBIJjp790lZ0>EJwgnxi zo9Q4nah>KlU@Fgmqxo-4hK-4Zg-gf4*2)^lwMIG)mL>pZwS+(r2zoWK1zUhE?QfdT z*o2ZbbuZ&yzl2!Pk}c8!n(W?@olS_W>_HEm*zx{;h@)74e;$gMBpQS0fW$jSG0D&& z=_1AQ^=0T=hW>pL;0%Fp>j)gvfAg(5*_c>=M>bt^b1O#(O0sgW2ixiV1-f2?ZA}bK z^na0+za#%m8OXbTr8c->Ahq>0;pP^Mv`;8LmK3JW5+ZtxhaZ4Z!nmx_R1CA0t+dQm z@EC`Lv=WTgmq&eAnpBznvaTbhmu4-m0FKLlA-wC0CkcTa14$!eRYPJEU3)4x;mc); z2#zeGef)aU^>t?)zEka6ich` z7Ye*L%%fvtu%XxdiJHwO%SqS{G-}4o> z+{01OZD&WNWyD%9P`CcvR@_y#z0_44ky?^VhZaNSR$qa0-#={^e@}N;ge>=Ss;U0Z zoO07%3CTz}WXf6mL%5p~x=QRr9RQLo0zpjq&JwbKn7CM2|48^33VaRLx3{v@u>(H9 zb~lCf3m-6T1EKrwx&#VS5+o>u@=zBeTat5k+nPuVRK+~mIn;vsZ zSWqic4wEv5F}?WwqzBt5!N<_7#S5847SSYgdmEun#k>`yRA!@Xjf~`_YTry{_(HUk z8#$rbgsOMC3f=K#?u>9Fi^W{L4gSgJFZM(Y*557sUv4FAT3b@^!t)+^2ok}6H!sCb zDJWOHF05#O-TC&6IksU`(lX-WsNuo$14`xl57dgV87sLIDE$|{{dDBCv3S1n*dgg# zDca|1+&mmA+8}Y-7~CHxGXrPubgWI@GpM*iDVQrS>_FHkcb_VhYIIYp0J|nrS^)InDAO#g5WkX+7&xBNx*a zDR2@`uR<>jGksDLJh$Q|H1?BmSMRciz^0CYhFVm)yLm9P9|1K*-hc~(zzIwc@FtlK zb-9c5>{fMrP5vN04$vv zg48XOP97{n^#)#XQ^c$tmN#$*yLDmI^h-0U^XH;Uu#j)9}HAk&mY24#j{ zX<%+FSEG39WN$za3KB4cd;XD<3)ZftQdeM~7MmOKy{NsHZ@5S1$jV{QX^Wx~LFBV9aIkI~`juh2X zf4*sW#ElF55`aI5twRFTZk&UAU9Z z$!o+nSysJ_6M~9(&J3FxFVDiR9IllUe=aH!X;jsZfC34FCJFP`E_(_(0wH?0^H`C+|MRv)%mps1B!tnAPa!sK*Y zUhIVRdW^h^y;3cvSry>a5v$8Kf|n>p7#ksGO~_2nL=N>6@2wrbCZbW#0^6>pZq0ec z>P|xQk4T%xoTDDFD_A#gF>R7S5R!hRfOF>xk`2TO;$UV1vHVU$_PX{CzbyJ!Jq0wC zZ`m^hD)du}(IbmV_KL099wFI0kVmw!Kd@2y8!Y?+3Hv=VjLP!X_BL|XC#Oiq#|rnL zvX&#-Bji6g1(*p1u;Jp~LJo0XOV&GqFU|tgL^^jjK`thsE@EYaFs(XzCPqLP-CoBK zNH4$Zqd!F_LZCeo{-uW9xu%iB6yz|56J}OMKlie0yakMX6G68*mTs3=a$ME30CrZ4pWn} zIMt1>^xeAk;4ySzr*&}!;|e3A=o^fLP<#bw=Q{=ec-Nt4z2E@4N@XH_|doXrl_%IsLdn^b;s6cVew;P)~o8h z(E9pY3_{Rj!xPBe8v^IfkdciE#Kg+X#>#pNh5(v006XdY*5E(e0fV>{01tnIMpRn| z6dR>mu<>t8c+V(J&oJ0O6f`6Wm5xjf=fO*tAH^Dlz8S?@^s)AqC8Voj%6tbz_yGi@ zJG1Wqd<4W`1}p)>z8g8{+8WpbY#U(8|H;0)hx0?y;9G#jB;leSAA8T8#ZIuHx3x@J znG;685hhGI*?O}3Rp>r_g_mZ8|EW|8qZsLS!#2-s&5~F?mmq5QWCTpK(r#9N|KZH* z;tD0s5NyAIPo>|w+$?EcLY))yueC8`X)a)f6kT0Q;;9SdN||wMq*9p&X%@x$vsRqkJDEuC70Qcl)W zXvTeuuvH||nX>scaM`nkm~1?F&2yw-*Krb$*4F3khIqc#F?=YLB}nOK0Y7eKf_eV~ z)<T>5#UI`$$h zXH2$aRvI*ZG%rP_*=D^`on)V!*}~I=SB4ond@i~^WnWCNN8ZpsS+qDfR-UjUB$5_K zCiZn|m*cc2envUpBg|-sqQD@Xi`>SQabJeu9c`^$eX%PBk|yF7=7x3^&8x^E@zA9gAuNpxxgjt``_#&b9|tG+ zU3R@!+}SnnT6C|K5==(BMN8w3xDs0d{(nMG`YR|$NOdL%cJCz!{hCmTiMX$%Pb1r z99)&0Rfq*WS$s%^6t}0H>Sp`{tO4m{Yx+L}p+(jGF(fnFGf2<#GT?C4u|AGsDgfjOsulH;azGuARQF z0g(M142;0`05a9JG`gMbl0*N_cG;1r&%G2(7~jgO%Wp0Tk!qI7`I4eGH1hhixA0Hx zCM7XaqVT`JLO*?aeiAWCXuUSK>dXxnNT~*5*dnaEpi3F*?Dhn1anO8cF zpEjB^4$B>WjHr#surK!d|C;M0^JsSgL&e%wVkrdYvbiU< zKZTr|Umn3VUiHJfpou281}x_@Zi33pK{V+74OGb0MT{i&>Den3^{n_F4_~On*DbGQ z(fcNT23DfcjwO=5K=u1Rl>BwX!biKE%TORimPfF@?Ha^7=M<-FUd6}wDs8Fw(U;wy zVRH}Sa0j{ewVH%do!%u8lmr)!1c9YouEU?a)!OB4HFRS7pn5SlO6C?6++i(mOM^ zva$nP{+)w<^^#2OtSoOb2+5&k(!hnjJ$>HMgN5P@O)=H6r{5$~ZZ zD>P&$6+ib>?KgX+@pQe<3DcWy+!uaspqApZMM>;Fc5-=GR?fvr)b-@pEen<|u3Rw| zzTJ|cb-3x^(Eg%b6)l^hZoBJDg(UBo*ZxB(>QpE^`x&g`a>c{brBXhB5#3|9>bjGl zQO8;QVbO0GG@6nx(p?JH2&vVm^K4$bBHUE2FzJ$_JfKtv5p!p#3@DeGi3Rw)h4cT< zN&PO-QL0};Qbub3SCE7%P`DKuA;5oBkK7M^DgfP#f#DY3JGQVDguqoj;M^IuazKg} zz+Qe=<&fgj#M018N8ifa!2+<7|5lFQMHUBCL}_{&pM1GnQp~Sclb}K=<`0o2?N&u0^#00H9)dE@!cx})Y-g>j z4-lgER#xV}{(jTsJEDe6fu4Nl?hE1WJilWkm-0>iZd0?TTlAUF0$7>&7s@YALOkju z(6STDg&E_~FV_+oSynEnHlw6xk)^exj2NlJGR{0fCh(*NLr*>ju&f;?zOB}SDN*Y{ zc%{j)1v4m_j|@v2di8+}OMrPHZ~q>zbBYH6L9ONzg`O0+@l%M_(eS`l#yi4kFT4*K zms*~<==fg);D^YxZQwFU17+jF^-|OK+sela*DzfMBtY+|$+LMxSYHe;v}*qR6p0d? zO*a~xUP;<6+*#vA@Vz#!G5P(5OZw(#rPg3HZqORjlww>LRLJ&1X(y}S>sBg>sm(QA zt$~&Nh$m9oK{~JtpHq|_;C&nsLo;o(Ug>l4h9JFp!kdU(w=vJdcp+(BbvAn6@pRf< za%PP_&LX4$-o<{jB8NE_5n`62j0~!wv&>~`S25PXYRoLiQoC|fSxxHyZ*Fm z?CcFptRQxH^RoP8`+%|&z2Wd-qIPXJ+EN}galZgkl#n2z*DN{;+88wMN^?QDbr||7 zCwB!mOAK7)qs(RD8Bw1nr%#rSJc-c@5#D|K+Jx09FU8`2&q2!?kK)a})W?4LtWZ12 z&Iw-MiA=l+jj=~05@t?t7auj#jMP|+Xf?MUtG~wbQt0EdLhY>B2tbGc0Rxl9fZp7UCV*;-NQ}!b0(fZ-mLofx?Ohl>7z2Go{NiHBgVu5 zkE*l*{eXe<>h5~t1RR%x>36~Tw_E7i>RS9F9&Vtwo7y7^sz>$&q1-!tBm8X#ms4r` zNuIZ|$~%0a05sEUk^+R91@I%hcZAOeT6;g#5IHG)fD3Uo5aQr^X0qb2O0JzIb8t9? zFR8>ov%G|VHXMN(u3twzyl(Yu=~z1HC29*0fSz!tw3l+CwR^4M7hVGv@%uPk@Hh53@g7rRm|Zt-Sl zrM<4ohMQ)t+HT~u>0)V5HP)qmowG67NRPNdTE-%YM5LasltNCiN6Y2mCsYz&CD@N_ z@N6Gx*$7_fA2#c@QGgg7q(nreI9O`Wh?}?+MQ~aPmypoHG$S(U8ja{Q(t8!KnqqR5 z`;3A}k5Nx)N3)U>x3Ze9;CfQKW52(-72auyM&|ATN*n1hIp- zfKC=G(6oYtK%iDs@hlVcxH z@iALY9k`;^#q0X&WLM$*cyL?0byOPm4EClWZwhead;s+R6*zY;sJMW^H%=B7wm*$T zfCbXmfsosf7S*2|YY15Qf4c#T_Yl_mp3v!=j;yChrQdzKm3>J&Ir+Ur631gIg*_I< z3GtE`VjVx$vp)G0twDdhuq_H=vNVLMfp!B0`E$e-#(yD!&XDU<$Z)K{tB);}#$Rar z_V7f~rA2in+Swh4eJKV_{Af={ER4EponnV8^D|Bp_?3?*^B|?K5cRv2u+wJbh_Y(q zVM7FE&!4Gjoa{;=4SKN$uyyXDr>nPkGgl<+hq-`uC;$tU2L8O0g#v>ZOkAuSz__P_ zCB#AvbpE#Cn?Wl&tOc04L3z9D8aN#34bzg5GpO{aGSQggvk1|nsA!yO>Amfu0Ag%d zyPr*~*OA$uEIvRTcgVMr2f~mJW!|HZBBfTcOK-8yo@jCMZ=&q-{K{ocqgH@P+;)WR z#&Y%nhN(MW99dg{p}HXkF}eCFeS}O-&YZ$KGXf&KS844SA7_V@53S}mnJ(L!MbY#ni8|Ug)!CRh6*7AlXk#Sx}#2j;a3(Wb^ycw9)bO< zkqf~szg+h}vf=*?8f#BVx;z+CR1sQpi>nIc_E z_b=I0{*7#0Kulu&y|JYOl$Hj7LqO;S2u}NlF!X;T-FRR`diz=Mcze`%`oEM8Sxear z0;3^i*T1+qc7RLZV*Q_9&M&n2Ul|jg-ZErQ$TZbL3FrKkH$_oQEo-k~UTsS$_VM*Q ze3wHkXoK9S?P>%)ObcG0Xk^!=Z?uo!4{lKsIi$Ot%eyr+_wv6cs)V1CYYWnOY0K8R zG|GG5UzbUpEtuzK_-b8E-{JxsGX(nXSPURWn1G2^mOnW9@6nl?j+rb!VhRjNxu4Q{ z%RF4C+Om-`u_Y>I635g92fs>GG$;}`Bl&SEp)~ukoQEzf*3NRUhv)R|<0l9j_R0^x zCm#pgE4)YD@XFIo*&(J(s33jIgk<3P>iflqS;aw(FxU;$?Fx=|oy72y(<|$;Raz6{ zf-M2N1p1C$P%hVJsGIqp*gcbX!2;HT* zU{o|G^Zk+;gHp9{7*pB!bsx{3x6#mjLEn&f2|h)=E+#|7hRE%^V{K7Cx1FVAtz1v3 z9-=qzja?iw8X}%9f$xKu6%f&5et5*iYeK0e0ww?|M9y0-epzYCmya3uF|9;-G|JYUB*9kdi0fjRG! z0|d#{gB32-#VLgJ^3{)H%~gQmCTd$|vZC;~oe`pWa-__RiV8tjQ$jH|qi(1fW!n$s zDalds)Ja4`yVIuw*dSyKD35R0uq>_};}79h5n!V)G{irCG8gH1n7sx&>e;q#5#iZc^XM(h%s6{eyZ3 z!lZ(Lp-@OWM3ABSm5l3Al;Kl*sj&!2U039)0aG=(vi;KQHy4ke>^vbR>IOR_odLc`f*ubQsNo0Ga9qhT>$ifVdN`gtDlw z+7yvF&uGOtEL!5A^WqK%ksn%Q-r0{`@9~ufAw_PfiJ7gW5{FClmx)97i~Iy!vlx)R z>K}6IN{fUU^?opX*Xr?OW~JsLE$llPxXw44>M>s3TQW^_OrDiMmE%bss{-OZZk@56 ztdNCDx2-MQn{ninfqm&5P$VayNZq@dDI}0U4D~l@@)uR|zblkKZL)t*Ylz$sWCpY8 zYl)MSd|VEZqqqgUTI6(x0nlWcwC^4dv?~R6JmM{0Qg?i+cRPRc{Bvw}*zq8$^o-WF zECrG>EVecX8lwQd3y)*I`eY&6%95AxqBS(bVx!52%p1?kLrzKLL$`=m_my!cF@;-Z z!X*37iO06!W*t-Vw#DJ27K!kn4vWZq9zx1i-`%#ItTG>toFr`ILx;<_q~lVuAypQ* z_?QSp@L(aW|H9Cs3)baC^l3s+yhS{2RpvR23EE4apr#Op`Zq7M{dGgya-F+g^ex?dM{g2tBLF zS}y3qYm(A%(})~qj%FYQF9b(x0OwAzjtK+^0s;Y4{V#8{1sg#6F1mkXqJJJ7`M*IA ztLfuW;|&r0?Eo_RN3!ZxiB@pGXY4bOV~c?CkGle3CcrZQrV9A^B^FTc{VWg;qCjGD#e5yC%cb6%Imjm|QCT?#Vb(*k9ptvAi9%;?Y@p}_m$ zyh3^HVSU?!7phT6OGBX) z4oR^S_4^Xl+bV@Hs5?f@7lChBaedqHD_sfFSwe<$D6^R4LnC2Oj%J01$#W%az7cZt zWQR);oyER+JHhnq8Dq@MSU%6LR>;WpT&#&oFiwaB_%%W0ur=RFQ|bpNL0G@mI`D;4 zuuoD~d~U?iflk?A&ACW^hW*DjAFe{ALYd%>6yB@DraB~<18Ht1FA5_ieXPz4v*4d2 zu$oIc@rNWg9i#!wA@&|{3>Lte?(Xe>Ea~^Q$X}N9d${VRw`X# zg6RrrNuDvXrEnspP(*en6vF-@oDqTpZ&JSZQG6tq&hrw|qKATiOuZGceO3_9-_~?fRLo@eG;?s*hs}*lVF| zjLqKYgXc zLl#n2{2B2FW_W)MVcgWj9m2z{9_`Ipob>V5(edSoGszE{|2&VQ2xB+U2;Aa25EJe? ziwML70Mr2-D60-X06eU0f$?KIVD;ypP)C-MmJ9@ae^p_9Zh^!i;db<{%oP7Tu8?de zFV9-(2}v~C%H?qkv!cCSDZLu=tgDNq;|^E9VKWa=MRKIZ7=uw1zxLRaGkIg2>k1VB<2m zdTsw4U=1bRAj(vpRtTR;uOXfUN9WLq$x{Sg(!c-kZAGAkIYofl2@#{<;JMsTQ&C#sTl8^VMDV#P;+x|@KHmJ=L%of@_e4D+5 zgh5kVuB)I@=*2Duu#QqamMO-Shj&_@K%d>zI7g!8`&w@#wg%c%&g^FX8Ydr$5W(!( zBk?iAt3zY$^o}6RKOh(QJn*#pHvJiBc8spu%u+F*M`RZh%`EPYtel^)K z1@#L)1Xnzlxi)dTCzDbmJ}T43$`<8cFI#=snwX!mMu|{I&1$QnAzv&WMKAhX=d`7V zS;rxP=?RMPVxOH`aJ%`{ew1s_Gey{~_1XGyp0vv7i!I^ZaQekaY*_TCE_$uz3Ol`f zNcSK`l_WyS!dqwd%Uhh(iC&#dBcM~rBIli}Dkfk-2v|R4{vA#JP|$y9JK$}Yn-L@= z48qX?tsS@R{ zpbwmRP9AI*fW?XccVu}rQ1yUlSbfX?gE%Xd52EH)9RGtvj>jj}TibJN(r{yXQNJI4Qp_dubKx6?h zCJ?Jgfmo$$Y2xyGy}-nP)ZU8pSJWl_=h3(SuOpC`=|}Ai{rT+?{g(}A{n-DlCAxnP z1-}ls`Dh^c-kHN;2C*_RvvLBQEd*ymw&8xQpZ^l$AH7Ra3sRu`T0&PZ%_!~SQA-Q~ znQ;^7x+4$HTgvk?vq84d)u&=ilcdo=}TCyp-HN`+!c18_ugP2Ye@ce;uF7M;WP@g zu+?YkVYthhlfE%$Ge_ou>ko4B-+w?H%j9iGYx2zV`AR40OZZ6%%16FD!U~VGq)MZ> zQT}sC5PdnhSq|JDs5ye%#|p`L#GelyYV}-O1cuMmp*{-x3Tk5KVJl;lsexb3xTXa~ z=-4%4pPkm0?1_rQ4k=voV#{#8fZ4Wro>@%620IMaMB_rO0l1F)DA3VZ9$!CSDwWk@ z5-*sH@O?;2Kt=oRsY}dvg>|k?(G9zQCzH=A>jVZtB}R1`ypsB&r*|l-WYSv7|mcs8a3AS z@>aL9dhV@z9$@*C|0YyGjD?jM*l+)3EWbkVe_tCd3L8N7=EL>_LWS;0lV_17mp91d z8{*r5GnO9TJkFD!UIKVRmDa9Yv4_b+)%;RERLZPRwb0>y$gOw^^Te@#fRJ>Te!8M^`xr_ocwWo=G>p+Zy_DYux|JZI!74 zPz=MrQw$fRk^5&E$-xAu&g|`h;Q5aU%009NPz*qwTwu3pD-J7kNf%g@Ruz}O56kJW zMWPn4H{=UvpS3qlax8n0%sdinrbfzkTh9_%q<*cW; zEj7GA4+MEYveC+Zz(Y_}1k|JGm4M{e44trJu#0U=FP}|2eSmyvH0=p1AwYaHb-RI) z!DfJp>VcyRoI8P5$lw`dn&fxo2U+BSU_3ow+x^!HESu`o-W1 zx9xzpC9kPdlili({!_dPBP4iwxTyju)Ktc|fLk~eL%TD>2CCmk4`FN|w zMLqYuHvt$z6A)5W|AnN?AT|)_cS8LQ$bb9Fe~qfiiaJ(_AVRmV3aZb^G2?%)Vq)cc z1>@zpm*f3p6J85NFf?txGKaCK@CzivL}O~MXL+%>{TfjP`B}_$K4cXW8?uUNYm(RsLSS(pUAY0 zyO}Bvai|6A;;@Duvu|K+&!v0|+KX!g(Whp4M}0=dN9PM~d0WriWvECOZA2c`#j_pT z!{wVN9)~qJK^G5c?v;nP!Kve@;?)Fy1~0u~We{av*?K|##tAX<3}vZr072u~84q8{ zBqEA1l>0TOBCaR#amT1~A;NUYRDh<*RBHes-usiplDTV)H!;~{hL;wmdU%;YMnHEh&Vr9m2DT z9hVlJAgvI(Is`f@VueR(llz?dnFao& z@WA%7v|P+I{j0eMpX?0ek+{~UDrv6Qtmcaui6%spXHaAP)*}e0!_Kh<*%`*ALGx6{ zw~*z0Rv8j5FjC2j{(Dp4j^7@LiXg5(4>}lt&B69S#}nB1`TelPU;hBvNBiUM~OvoXItYKe2~t6nDuCmC-UDIJ~r{e?|hCureMFqmER@Y86b zajBml|O}wj@-wP31C4UT^iG~4DEYfmA5aaqh9=a|NE)%26<^HoN{@GMy2@W zgMc*lWJbrL28ZBxA34xw+&B4&_iuv%=y(4CtkUmL5$a36|rz7er)lyWSkXF6QogdZ?wiX&@QJhfJOBlW9EWbd*B=htWYuNUK2WWt zAb8kE7Bi-kF@P-dwH%pmyt_EkOImm6nQM}m9x~Z{>{o+i4((09diRQo?JdvV z+@3bEr~uR@ceWCHM;TvMPq?BZ;W+s4y2uCKk+v9)8l^{E+LgzY_%z`K z$@{k!FJ6kA`($_+_|2H!U)1kR;98aOjKri$dEk+DN~CJJ{WCt!QbeUgO#{b$f3iQ@ zEhZ(Z>nGD3FAN-Zb(XB)$}RSc2F)@B6Ub4bKp}nCxiiSn851ij8yo1az~W$Splg5Y z!Ko~{GD`sK&z}cU#*oRgIBXf@(vIKu?F#c!UNo zxL15Eb9l_{1@9lv&&)KN-#KoaZ|jvfsyCJoRdc^p*&^eJ2A+5Wi;EeRU_!l+A(6yp zq`5uaX(~hup&?%2zEX6qgndgNC%H$wy!KHQalOksZPs4Rbo(z{Ay2J#_V>9QWLG@a z1x}crBbxMzIZc1wE$+MMB1^--ZO)Hw%hNEZarh~-(xb~xY{UCrJ|&EFnZuD`vtkDU zRSZ3X%m+82<@M+5!5JRT!CT!S3R$#N2oA~y&K(yZ*+D=T6H;!o{b`zm?R5dg>*_ez z{{Kkrw~s~yU#9<6L5`*q{w^Vj80`Io^ETo;$c14iqF0S_o=}(m=cd6YJ*;TRR=z!O z?pP;7q=Y=e!Unu404XbfZ;t?4$$@eWmaH-;d46ESb)>PmqN_Q}aiB8UQ1p zLDSfX+|*Iy(sr+03U`A;;U%5T+u?4E6p}>doS5uBrP!-$Ct1We_i_bt}0I#4JK42|9P(1H%d)Oj1rq7ieP_S|$tZ3Mivt;$W3+ z7@r=oRL#C#_MpZQ*Y{oIx;i^Zro-tL2bW_$CVXJF(wGr#HlyK7l>NzOvOPUGwI9*< zha{h`?>X~RX1u}GGAsPUr(!^eISq8MU9*U$Mk8Ssy?SigGv#Hdiu$rd6qcuKeSLz7No#zMij1x= z#O=6o``v`L3G2teJDx%6cHC&lbG$e48;%K1&TGI0hX0)#Z~&Yo2zVJ6;s(DSrvM%? z_`5c=`r{7}0CW@8-GQk`6O-zPlc1&MtgEHsytwfF@q^maQ(o>3wXBt!jj5QJp|(p| zgttm*b84h_P(h-8u1tWpi$hTz8*VIvE_bVZ5c3#wB$fp8;i14mg~DNX)%{y8A@#^T z9UpLs65!miEdz0h-+}EHiv%E66LVk*!x4C1%EZ9wrid0{{4&r?z_W0dfw3-L?QD9Db7Gs|oZuV`M8u|I27xe3K z2KV}jL3B=EvU)*_NdtC=$k#C;#+%r=MJ5F^dhto`wzNd(n)sy|>FOWGJQKnr%6)~~ z4fp<1nA4>2G_mr&idy7U(#mX3K+3BJ1uuCz4+ zau!qQ!N8o^&=5J2gc55*Wxjsx((wu{Pco@a9Kx%UkKQ><*~3l6j<332b88-c+^VzSALqVB$@giK(Q1{?!G-2qUk0SLLEKyfrPfsyNhNwH3jj4u^dLSvI zQ0O^x2RyV?oO!@crYEt7v9?aa2hvFMCWjG**d>0aug7%T?nwiu!m$S5?$qN)j%CYN z)Y%>)MqNz!)t9NJt>=Mgi7B3}aVg5{>*Fp~ooz}9n$(f8ht0j;AOUEfRO^!gmUse4 z#r6OAGXM}n04T;kQgJ(5D|5qtI^$p3dcXe193U9;t8e)Ct*50~Oodvte? z5>hl6UlG`C37bGECKpm5FsT5UqY|S2pkSjMXiVT(a?KxjvNsE ziOXFJ54WYh7#hoF=1p+4M=|l4LmKpD-h!{=Po^M&4xeAG^9TrXwf$J87*;a$1Q#g9 zoxd&%X>V%Sj|o-cp!7W-gC(51&Xh{)F$)yCCL+y}U^0RmD;>70EQD?H4GEzu3>s4; zt`r@6e*~7igW_HgRWEaL44SQ-jMn_IaXq?$!c33~W9;p_NNnie@$pS>D%>jK)X%X; zWGQcfOKXx#iooG^d|(uzeBiqItiE<@$=F~=qxTuVECAV+cZ~cnQd<6PJREXi%TcF$ zVCay}f6hspc=g9(rlOcSLS|{|@n>7sClJk}HS{zim~HLuw_y~^)m5|TbFxLa?IM`0 zBnapfPkf|ZrVuS3z0UcOUe3fxwl}ZGcdl~@)qnTONw=8xKWUK70L%D8Yz5$Q0FX`q z1%rR!$bQM8{iV5T3xF&CN1d5U03`RHWxFhx74pL`M-Z?lFkC_G%MM80a0~EW&lO|B zn6Nk|Hg8?`FG7A+pzcn^NhvHPOad*;FwN#5*zpNrCt5P|pC`T@UtgB^7I6mJ8{r8j zqzM}cUUU~hl}&N(cY(HiSXRQ5M{ib#+Vr?`bQ?`ULUMQTcT%11%9EB<%CW8ppIjuJ zZ~(z}bz>~PgKe9qsd;6d+Z(( zhLN$IgE2rt36Q$~Z#kuZ*CMF=*e%t^fCjiY_`D5MSv(%$)}Qp!s+Lukd4pE?-d8v! zqa@v4L{+wxs2zs;3k*p!U7OUwUHRk4E%AIs8I9Va0D8zoL83g6ES8jkzkC&#eMdIyx|L#>!j z)pq(WdF8v1!+m%8P){A|Pwi}CW?wK>Mh57wq^hUwO6{k&?rQRzpj5TD^aT zdaplBYq-p*vh3=_HK<|vpavV;ij;0lq{x=9P?(I4;w#2yyJPY40W!C6_EvVLn)WrK zk{|bRf9JOk@0jz(_0wy=0oET@4u5`lRt8Q+CT0L~>=)nqmudcwECB#R{}YY(cQ4ft z%?;Db04Ly@E#!6&&BfWkP05Dl&SsGh7Pum?J#%vJ=r8Mi9eAGsTCL5t9T9#haS68z z8L3z6EQn``lt_mPz1f)rP5^?(oW$4-Ia+p7f7c|R3~4(%Aaml}F=<`Pup2o|E|E%z zE&_GA3yU&V(S{dk{}uCf!RNPkik{Ld^yf~0<_H>q^+)d%V8?(wlq!y+s$Ai|eILRTQRyxz2?_-pq627H^{uweZZoecQ7 zmp|VRJD}jj3SguD%6nV@@z2i=wJv}GVdrcBcz%9rjTk%F065qG*DNSf5THAW27aHh z9po*yb=`VUZ<4X~+aw=W618MG5I(GZq(`M?X@k&1%==(rMZTq{ZZTjSX@O>fVX5Jd^ZM zAC~bPLce23PUzp_dM#Xtpr1^JpT_-%ePY1r02=s@@=fC#G0^r~M z=! zgGAMy?`_vx(%gh#;uO~^ASOhpw9kK18}$)ujHK&b0CbC5feYE+<%m?~;1=aVT~n0r zH28TxQq3b}qaOQVnAG__!(WZISO819LswGdiGpY(DYN_ZYWD1dXe>RaCzVw|Rf&s% zcnY61Hff$l+d^iblIXmVZW*$3 z$e9EeK2CawU3j{#i}XGnLe;nEUEUhsBBt8B*$$xef|23%lcL7VG!YH85`IBFZK~9vYXnQpLzDmSW+Cn;! zBAg~LGc*`z`+cFWE+&+@{0yFfIel9_cl}1K^;Uw^TzD9Ta{YuP4$L@O1ne0=shfFHSVj#Y?!% zys@8sd}1$exRAk#Uo9bQT9HhsR7@`+rS`(!?HWD&z&w10g~ce0(^7qQ^2faFfdWLB zsE0kow^8D;IF_0YhsD5JQ>!rG%wEmKgNH?edECDT*m2y(?LU!@Kc&=61hEJ^ehS{~w+KFlzaKZQ2925HJ=nmodhZ6Zxxz^}pk@g?;d$ zR={u92RMm8uD%1lJV5h@nT7G+$ou?~fcqx`>bH7aRBRgpAo&yZ>@=uL#PIT;A3+#k zpfkcosbr%nS0~W(3;v2pFb#oP+|Jo@VA6xMTj&UH>xzjfVZhSmAPW`Xbr7n8QC?@Z z`3a~LO@tDi8a@W~_I&p)v!iAIcy(bh@wO0XfORr}5N~CasS5WXMtS@34V=rAI5};Q zlG~&qr2YiFue5tlNH_pDC2}%7pc@PR80xy3&^vEA#Ov%q{j-I<#1IrG(_z3NGJ0y9 zFxoIr983u2+HaAaT`6g;4dA1V{&xq@0jQ$@zS^&#)=D4n`v0nI|B?&+CExhJQJ8;X zzmnsUw30H=zsEm{zZpVC9{_UPw;MNBrv&I*$muR_A&GAS!5hi|^fR=3Fg}|7t_HP_b>+?(0vCy9X|zx8I3NWTf=2s z48gJcwQC+>XCa97&tq;ns`BOGWEx%lVQI{Xpn1u!Dj@dXi=$yhgUj%@+7{g=JoX+@ z_DM{BKKLf;`)Y9;v0jItMN)ZXcmA8sEAt%F?axwd1Job)`F?sD04W<_^Q`|fnoZZp z+{EPXn>b4VD7^jzB)x|B#{+`~;d?T*!Yl9xRK;c`McIfN%snC>f4vw;!HN=#&wxP( zQ`a@!NOjM6z#F0@&RP3~CHaO#g*%{LDpwiF@!-)+g}zS}0iOQk*ChitpMEEMPjv zwkUDNbG+b*yQS0=NX+L(k$97~PgE(6MjH!)eycBwmM7UbgCbIeHlD5dO!{u5l~*sr%6yo) zxQtFcdkfSDB$wCfl4BX0-E*h;!6;6}9;?|HGrxfA#q>~)c@D>#USK@sqIgi&Y}dTj zmrOZ|=h2a@1VSGv_fm9TzlLwzA#R@o(dFi~>PGYYY%IvT_N-wxkE*X!v@Eylb}8G- zj0Lt&!x+cVQoCgEN^9W5+bnZadFQBLdOI$7Q`4mIybA@tATy&r=<;p_C|X0>n!LxE z;nv_$)iuWPWSZs5Rcw6(uf6ffZ%xRa*3)0E{O2R|=dK*k{so|@|G~Kiz>*Ag?eqZ( zwg0!A^bcQxX9>z<%bjOY2!0YffH0n)Qm-!^Z;Tg4%mGtl0@Rw$xmXa zJ%R3pR~QU7c!0Be2lh=Dg0lCC`P+(62j;`xslpWoqla*Si?pWm0D);6PQ!|5&cbJs z!yA#kVa^}U=(CogoPq&nfz49+TrZwPY5|Dk@)-3F&3?o{sz6i_beVa85uL;A*vnlx zJRhEQz(1rz(@=YFWi@Q5jLhiG4eOQOz^7Z~(%PeyOVHj&X4+JfE@V75g>C-k`LV>= zp&$V6BM`9k0pI%1o_s*k_v;@38SLonVEnIs@n3UbfZ+gOe1G>UfU!SvJpzCAcd`g` z;q`djg-W3%fr-n%Em$q*zB6}CU#A`V>Iy7MKVi;vz3)knIw;X_LX+FDAK1ue8dO=@ zCKkxI><+mww4|6sd*xc!zEgQ)^@YK6Mv^}IJyk%hR$US+4&9k?%&QSg3qj1FYyO8l zh0^E2zR|oQ;GUsw)zePe5L2yb_wNX&weec90z=s1yYjQAi{F5{HR}QSXw1@j z6lLZO$z&;1W2|oul{Sw;x9|Y@vmO6Eg=0*UVj)L`{hDVp8mKI31}pfybU4)IO&hAHL%e-+$NF&}Am^JMR&3R$X1* zXkI+;)`@gRC07ro9@U96Hf}}?|NO$dsVy1`!VN{AZW)muWjA&$R((&k+J}U(1WLZ~ z&>Ty_G!HJer}X~iYo<^lDU6KuI^m0ZHql}l)qKu0gv^IebzJ31a1h+0e5AU6&vDXg zwKRVg%6`UFe_qQ5FmVAGBY@W0U#Qogl!$*0vC=hi{%e5j-#rF80Q>P%AEK+RTCM#) zKsE`b+d|$2BcC*2M&Zeq^`MjrNj%r+#Q_P`Y#vI5rNnPX2kK+nQC^L~8WZGu(l4MB zD_t?mz=pB(f;<27NRW~Hg6cm)^ArexCnknQn| zAX)CboPOA~JeH!+QYqWk7Y&aH8BvPi;r~z_)o6#mKV?w3t)_z{4guHK1!7N>5+S{X z_=|sr;CabsB0swlhGubLnbgiXioJIOn&pqT%8b&G3-X^rzCRTo9;zda*gGao0U}s9^ zV1MSAOf&AX(0^AW71=53@9RnO_Bux$tZHt|AT#Yi+q&5v!t*N-{G z$Dg)ZdR|M5IorV~^*B01eLBC#8dJb?OW8(%K(_Yp%C1|n+RZ;HtXjZQ9~ts)(J?;l zK%6^TyS%{Z2;$a zCL-=LRTM02vN?hN>1YEO#zX_zsgWg@!wCWOi=+`MJH4pg(8u{(!0Md|83~+!nL=L(29;~YDjE+hI&xHKli&TwL>>5Gp zeOj;(BTt+!I9*LaDv49ZP(4mc2y0$fjqmrvTuru)@-@t|VkF{QRQg^CU%u>;#^ZCz z8C8vg-)_!5JahvNU@usJ#SB>guOQ-o)Qtcj5JO8kCtGVPYKjs1QE5q9g&Jjfsv&wh zNqQ+-y0MS{YbXo~#^%i{J`j*t84wWBfBI8@l~3sG9Lyc9_0M=M9oGgvXLo!xu%K2Z zL5-`6FE4N1MM;aJ2w|iw6&YzskZJ&@3;v*{rWXiVDl_r+^2lflSFnu6f(Px!(j3jGStBgJ_K5Vc*ZJzo@w!ihhE3hTrXM)KxmZsRBLV<38<% zxvdn*m%0&ckVK)8FEa&VXbD}%3nk1O)X^xW_CASaWn@fXJ@T{+kvg9S_)OL*Ww0u% z(DXHZKWHj#mtJXKwppcF-&h82HomeSsM!|PAsYv7FCQH(x<&z3l}y}i88c0MHl?;S zo3s}sNS}8|GK4jU$ufbx?L2muv(DypeygO509`%(5|lbJBEJU#!EUEo6=!1W${B7w)j|YZ<> z=ttQ$q70RipI;){s!Z<^ujBUb*FAxEYd^Ktp^velmG$F=Zd0`+<+a4+$>uooXi0DL zk8qA9iv38_?X$yP0`?-ois;+Df;6ai!YBuqBCtUyPfQ&nzdJl*}Ic%AmyAy*?Id7HN3jLfpCPKX2~ zSCpZgvEm#oX7D@-CEPe;yAEUvg#2oUOAmRHDiy?ZG_JkT%fLp*W*;5F%73RYca;d# zWeZe`oYM|UV^NT{-*|Xb`&k3yJf+(=>H?}Sn%A*=oF=Vn4LKy4b8GyXw05@;+cfA> zm9vjyAcU|leS>ko+Qkd?xx?KHn?`o}vpye#=)Pnk1Zo#S+=JG_t<}fGvoaM4eO}1G z%X8iZ9Ua2K%u1t%v11(dPv+kRPwS0wZ}u7<1YY=%HdUC&i)y@6hj}XBC^q`b+`dv% zxl>ZB3n)J6x%i0g?!-%pmwBl07JT zdo>?1OCFQ_KK(`b!g2~#5g$ciw7vGhJ%oN6<1Iz! z7kTf#ny^Wc!9)Cv0pz>@Fsw8#mV~zL!A2Va)$-Iv@Jg290i>xupPEKcN>~wim5{HY z@r{)AAZ8BGrz?qdIIJJ!$mnlv9?QQ4OXq3B@G=-+>>CR*u^9(gNp<&FVM4U~KS(k& zq-#!*h%)?uz-ClZD5#%vsPoa0%E3qEMbak2wJV9Dp^;2h|M+C3RGqanihaCTUp9sg zFS2|Mg@9jM=6QOoS$i8QkM(8riT2uO)oTihfUWnNrOpIDfLTSW4~1&_gU(;2fF1^4 zsKSc~m`NiTpwY%CpvD_jVZvYrA8Aa>5^^NOHBc2j!%^zswWg zfSv9mZ6}v!QNwY|4MXR1U8c+d%Aap!x1=G-_qf(voThZ9f9F=NHu)5 z+)hJRFG}!Dz9TuWRHJ2dEkcD2`%!6$$X-+=+T^MIJsWXYz zclW{a?7{i!rD%O6|3yV)6(hU1)0uQL%cbc8l4nbJWzd6`*zu@@AcfR8p)lMG9{KpY zeF3JfEtr}zd3jeEi$EQ2$5MOsV;Ub)SCQ=9>7ax|d)W6MJ{a>>*P&kuPQRQ&H7}W3 z0*ml#N^bMeD2a$vaS##qiS=}SQ@uMvr+kOco_Cc>&%zeKZrI(h_8os5mPo-UaO++) z(j(HTxzYy%!9`qgRju0%omK(!8S-R5N~5Y_Cz0^jP?m`3I3u*mLA35$&qB2;SyOGo zxefWW5Ms7|Z}I3}22sp5fzYM{}FeFD2qZ~Xg=k8K*PE?m1DeB+@yCX2l_YjIOwRh1iuV64zU zU2m>!x&uqf@PooC*MP*#&v>KY=6)~=RB6b`@sl5a8%ZBggP-#B`0~aViv@~lI3D2t zF?0ngZ&Ip8?rR592sAC^S62K;M#39Zc9I@b0kWG=mRkLNA$T8=RNXQrg0UArSpuGd z?VW4p();+Zvb8$=oKLNiNz_viFImGEtE2B^j++OCnE-z5y(1xZy>9IGaw>zF6LXS-WGwIcRuy@cqM@&GjX6Zip_8Irh76czc}8PV%%20fBA@ zRR+H8>9|myJFsM%*pFe{oy5E_*g7TJvxAC>FK zNP^Xn8=y$*XdmWPd41|XA}|S2BzIZ$03SMMMmg@jIJ8^#srygt4?jy0eX!O8BA8ph zj-{)PWz_6E=67|tT?bjuxh!;d-%C+-y+;q46n=>I5RzcK0IF?}*D{5_wyy1P&eQ}Ct1@NdzF;>5gVe3#`;W#v-gg=$MduNiMn|t7 zpFKyab4C@h#wh3gyD{o*8Q^AKpI>fX#ELB(rZ`r@q#BO1KEMk`GHI7uf z&Ki=v=(`LS(+6Foa9Q=>(ZlDHsPkq2A)Fzq>sl7t2QPbb?x>6Tyl#e@ff;xUdOJpg z<*NjA4t9X0`(5{gT2KM2#th@R7y2^B=^3A=x0s%bJ@!?dqYc%Cx?&kr&@)F`4;E&P zw>43;_hFG#xQDWeOa-Ty#6klJU(%QH`3m*wf=I%%;ATEpw1k=>eGh=ZDQ?z1W9)ZY^0G>M%-`cGKyd3$t8e2x-VuK{Q>P( z3-xhSEknme4lo{;=$UZe->^C3p6=7*Eo-1+)yIHF%Q}F9up}x!D+8BV>N1+hZC-`} zQ%+)e(aM0-pyh-4~`6F&!r8Z7Jfo;nh& zD+T_Begg~gx}mPJqI!f)?sE`P5ie$gIh|zs35n|`zN1d|sZNMCtunGO>z1!e zkokv*y2WO=FN=#SONIW-Q0vMZEDko7@-@v+5R4@{=sEB87qxg|&_$j;ge^>uZC0TOGqp}di|n$;cr`T zE#0!4g0g+#%j@At(m1HEdgVp+O5<}K#o_hfWhF>@S-ky2y4}mufk>wVd-c8c)z>#7 zpKH%8pK6_U4sI^YuMdGs$RVZ%8cmq9dO-L0_qMzsuGm2937b!=$S`KAOYdUEEpxD| zav}K4+8V{<{AN@x1VD;NDY~L1^tG`9JU}9r`WnS!kDK^o+48xiY`Od zP%YoUMCD&eZ_%AZSQX9fOR#J1NAF`np6eUSzR$4{7p@Oh9I|(-ft_p|L!|JkQC}5c zm$p)>vcf0vG0rS&bjm=brfF@Mfu;?rI5(nQ?p0Zv{#buu!1ELVLq_E7>0IZ$GGpOf z%cLUe)9(d>a(fC7H75U(LA=*y;zS@lSAc;_Bz=yQyQ4O|F>A>ruCdmT-C;_h(qCIF zo@_t1zwr$hfdto5{vPAj<-Hi2r(w#hHc=ZJ7!j}4^IRxHT?lOLaqYUTVY{a@zjWw~ zxAX(@?1Oo4N8aT|_b!i8?CyRudGhDuDN0;5bPosvc#OP=f{MI}K(kwNqb}5fN-Sw9 zBtgLfc@{7(tD&$fNISRqjezJ}SYG()1$Yj;JPB?Ds9QDuPdmY(ZJ2Y09dL~AXK(SG zC<}Iwp}4V{9`ofjpx-jKm%mK>c%#UmEfTnctf$1rhc`Is+(3FXxha+TY}<)&WP=>P zaTtZd+}=s)vKN66CW4Ih5f){8zr*Bw_MJdBj+h;qGK32=JIDjk#_m9Nfe>E6ER2* z+!blj=(`Qc@c`(sIAv?YI{i-TGECuf>7)0lgA(5uWzRUI1~X$6MTue0-CV+p69a-c zNF(c9?-qxd5U&YpNkz!P8O=W!F+K3^(J8d(GzSncx#tJELmyR5&6pUwOUQ3x_hcss zSgI8UTsjF?pL5!noX8^~0xhReuRhP1^!42(^hY{W=9m%3d+WB-;KrHcAUEX5KF2EG zT--Z!&`0~d7m8VKB5ITvv+8EpKdW2XH3&=X?Wkx07A0w_mAS#A<{eSZ&?8GOAApP! zo&b&o{}L1}9B{18+WpOL@tu>};u*?dJdkNgC5&EY7FD>gFmo89V}iVF2DbrXOXDAmAVtPuF@*aknz&R zI2jm~aY-wA-0u9j1h@kvNm>ge>Xv#kVoH-5^vNq9Q!SoYa?qcw)02O`Ha0i+1Pr0C zLTmJgvypy5jWu*xc)5C9QDTLqYGco&(YGhG$AP$-`SQB&F0W!-RUjE=tw1zd$m%&5nh7tnWiWV%$w-IvkgHucfE1g6_ z^Kk4c1Q(Za>4GPo;J~4Od)bW5TWGLnfy}BAF=_Fe6#^~pd5UZi?6c}c)TO$Rw-;7E z{T!F!B5|jr8yL|+y+AyZ#*{=q-M)+}P<->wCL=DxZDQ%v6VFi8P_Ry}S$s{-Y(*o# zU6g5UhMSLz;xVqLt`ZS?9r+Z6F)_@iSH}L20YqWM_24{dbAtT9#}|Fr5E^y44G}ta ze9*;;UnKUXc7!HA_RF4bkf1cP&rsX(J09^BF@9fTX(i@QiL-lVaZ?m{v9PO zLIQG-tDL)7iInFVNSZ^Y`%auljMPgL^*JiZEe9;r+euZvn&+v`6Ofwe@&n|=jf16U z67DsB5N(r>g9CPD0}OV7StFX8=CDGX=F)Mo`pq?YaeG@q40I9XMjhn6=ALG<8?k*en6H8h=TJw z57e+YKgUvQi$m1n-}GOl^qks-gO;V!en4~(wO|d+}zo!ndEIFPBN+&FMd-# zF~&fp$DXGX_^mbajO3dQ3Ar$P=y=jRKvExjEr-eMAqFtQgDUGtn-!tW;v$k~l_Oh6 zUPc}bDNtR28JH&9j_NJ1A+=uLO!zEmq&hh(CX7lKf1DU1MUhP{2-0dgG}R&sh~$ zWoM^~)f@zFPg}>fQLiEfn{R{dU|8@isnO1<_q}aHt$SY3*&ZRUO^<%lCKLf&8;lBJ zR<8iwB<$=Zzher*TgNgaFOUo$r&`z%4Yf+8>S&PyS7XEnY=lov{pX=1V6k@*BDGs$%GrI&A|^rM22mH(LWXS}1r3%IBd+!DE#9J3 z%ikm$x_l)op>=Lq@^A%`#(rXSRcsOldLO_A#2+R(-0kMZdyY_b=b#JA6qQ5vUo*EV>FR_41MK z-=PX`B}grGYw+>mAyr?CDZB}(lsBj<@HCneL-6^=sp~dzFyR>2Z+MNXy|Z(Pb1)RK zzClN??>8#+tfn)UXAuDs>Z99EHD#0cy}2&dKv{w0y;o5l5Vf8gDp%ifi%KiaE5zml z+fg0R(nhFoGWvKb`bRs|OI<50h>|t$USBV}jL|q&Lm+`x|MH=DmBV~Z z_lK}+2QG=Dl&f7Ky&k$JrsWF9;30;A|7_d@U2cl!@uUu^D}r^u-s;;kqM8`iSn9D5 z2rRHD;dR%xh3z6@H(&JP$Xq&QPbZ(;Hwm`EJ3p~Af1Y^;pllPE*RQmD;ujswSjoB zFn^rkKj{x@BYijgG(thiq#bCm<&wi=9!BcwiTf*S9RWLl>okgl?7~e^fKROdp8(atE zpp-XiBsOqfK?E6z9i29stu@*$HQKG+>22*M?Ij^_+T&}Ck%o&3+<#=hRxsfW|17)# zDSE^WBd>(|xU-04O8Zb_2_6W{>l>`h)4x55)D|Aw#m^WEno!x;f|dojW2`xDJE_e%+RDWtDPkcHZ>LdQizjs_mRT8AuSMg1{LUV{ zsN~3PpUfj0+12Ss{cLR(9N{|*YYd60Ca!_9i=GJo!(3N)wn2BCt!7h%^fL?(t+FVT zO6KMriu%R^?6EZ8ncPh#*bfjGsrR%#c`KldgGWDN&zC-~bNG?j2M)L4H_q(_^~Luz zlieaz2h;BQwP+OBalv)x-6GNUPR zRa;rZN5e+=3gWoI%O!!xWKG41-=cV0VrC1oNg8p&D055=^qcrPUQ>(zFqaM9%Rg_$LO& zYE0gMVi-#~1og1Dj>LU`OqY4(trl04>nsh(W+6>o)5T(J5#8x00|5wF%P)1%!S zq8aTQ;yqy533Ctr=q{l&km~eH9^y;+1lP^`txj!(MuVKn(1fWbBsZ=~H+uHsfzhb#NB#|b+^t-jFRA8C*%1?b#+Va)M(zotB`O!RsY0SMe&3Gsk`Wxc zH@`{RPq&?An%q*!Ep)Bn-XAY-C9&TNtIWyFPu!AviXkf?u$2Mm_<3mI)+#fd{Cu3t zx(ZhoE^SWp4k0;1fdQs(Kpn3^KFj6`EUL;aBl~X4KxM5tEap)?Y)Q}UDI=2Uc&pbf$nv>##F^*H!mgV3{zdih71@9Zm2K^w~T>; z$JeOUMK8JCtr=!d3>&XGW;DWWge$^G%Rx65Z7=%fy&#EWeu0W)YAs>ehD{pOHiBy1m7|Ae5lWUV=>-d`+N!vWISf(!VI4uX z89eEH3c~0E*16_mO`=xPq5}|yn_${Qaj<869Q23dBDKn7*4ZnRd#fBW9A#tRa4ZWW zk~w8Pk_oD5(MIIVAMa>LTFN$q+{teZlg{_|_=i9s?BTC~F)M8qt<}VH`;S2(%35%4 z3d{wX)D+Uh4(>umBO5u~xfj1<@7C!rVeN3Tn9)?%mhc+Bv1JS#Fo*ARq6wq&ECwjp z=~NLNHOvKci?C~cT{rkDuA{Tn$%>I8Qnh)u+)h3nY47~l2{twcJU<#9$2z5qO+4*I zzsRe`<(>{vBM(BmA_`O6JbcIILaCmT=kcv0y)Km-@-6@vc&PHuhdZh|RkJe1|JGq~ zetUHM!p&j0NJ-VZicu7EBOgLz1y2ogpO=Ib=5TRo?>3twUP8u(VkE6BG*W>Cg0m8+ zExub-JqN%4E=UU(bS_9ay_|)B1JuA{o=$*RLyOE&60Yg9CsrBjhedgg^hpXGSWy+p zGDV5YeL<$$WJ1OMlDp|FBwEMVK-6!y?0Ni+;GwKr>7^LOkL2Oma)jU!t41g6lsD?Kv=AP+DV>*wMW2FRQ2QEfXh`&bfQgu#?r5nXToM;N~O42qXvwKLEG%N)RP~368St4qomhl*{9VIA>#1 zDPEj>P`cVz*K?d%{jx#y1Qq<6?}(TSr?9>IE+ zx(*Bsj1~KY-lA%>K{OgST$!D4(;WPOvC)wp_izukiBEuJW-sDKkHUOK^`X=)oVHjr zwRpEBo53T8XKQ_(WOc@iiPT<-O!jl~Tq^Pb^eg4}$>_J-u~exq?WDAN=!}|A_%gQS zcu@=pFpRl79Q0;_TQV7@M`=aiWATHpM7fb7>auhXQ`qX=Y^2S1G~rwPq$e~5&EbR} z^4-+;S)72eAXlWtvwCFPP+D<~($|eNXmv@koIrSx`46s+pU3;d?jIknlMkvV+9MVx zCo+z{yq|pEqVrJ<*;pj8+v*165+enr)7$6c6stVJcldhiG$WrO3n+LG+f3DD%#EWH z)yYX+w`5h*$;qm+uuNCKHp1^yU9<7C1fp*y*tb}+yDFwj1;oDfoDQJ;YSZ~n^)@D{ zW$7Ez=F5|Wn$`N6eOVx!4Yaf>Ux>m^#ls^LA}59nnl7=0x3$j0m6PJE5r(GlQl-{6 zJcJ4AH2r#4CS3_>{A0ES8mZG3OC-VAeC&4U{j(KRen|Fk+!fPLT5{AleU*lxmoPvi z#bFmO%*h$@>g|3}czw6Kfb>PPe>0WGEnMR+i?F5NCD2`|vKTt%tT_bYs6k_r2>R^k(InwAV zD1lN(((w9R4xB4?^`|x|SJrULvg2UT+Pnkw38>lP_NY(FPAgA}izZ8=l!0Vr0k4ln>v`1a>T(&71NUYL53!%->BKLv$9t zrpfZqhoc_5vnl5yE)<5KWkX0_xvIRx%_F!Ql zw+D8L)-e2-8sOTnA33}RMkcmkK0C-n&z1xkUbnaOmeyYi-W55?+#m$4t{-GjC_q3M z6w25er1Zl%XEM3VTk{+dowM*ZRy}F~?r4aolUB#&`UVFN`RN&Lt?whVw{@x*q&Z~2 zd-?J_Lq5$*9?~iJ8hDSQ-#5Bus=aoIb2Wq&sNHh#QLSd{$y1&s!@8TSsH+mKk_9|S zZyXt#p?IkPHeqtO3e-c+f>lN;&ffe%2(BF^q`vC8$h>Wilc!R`cdGn8-wfH6=UG~g&f$68zL7sKz%Ku`ejnKO3sJF-EsfL=~m&sbPA_@rve{x{F-Jl_q%^2 zhRa|cbYI`dG+81ptx=EnS#$)=C1^zndJ3y0{Z(e^Ms~9vuVR`d3%Tvx(6b6z?#hTDY$Von7HO(wsWg~bcP@Y1=)3LmE}~iPk$5!P3b;mVxZQYW}c^@ z49&&9F_$@ISd)si3A~rdLY0F9RMc5@FGqOm#`po2zyv3J+Y{O{-jDK~j*);&B(>PV z^d69J+E2(>LlrbxCb(&`PIp0w5dAI^ zo%X{)*_($pc$e6~m@Hx4pCOC+wu;^tBNZ^8Q!1@-bcI7LO@xeqStmXBs>3Fwk~U>l zA2^12s6v*Si`VvF)C?@lTwF9d=IU=&-p$mC5{>CL)9|D0|k>TJ#OuV$8LEpbZzRPYS?sxuiCBcpf4}8PY z?TByTE5X7$M4rA;+N{rDBVF8;5P7NcLrhVXgM^MQQS%C3j11!0wPSa(p5&4NN3~tl zn;Yzbwa#(<2(>X+5U=0A>KhUf6ljl<$?(Z`Q!iuN7)b; z$jpI9Phou(m`k}}p;LuC|F6Jwx%iRGT{SKtbhKP&2WYiw)ho9#0}P4U=He)bD|Sz> zR8OU+*(Mv29S=ON#3mcT`zI-keTpIw&()9`F@iPos0Z`eCpU~#5`1Yx(>dZj96(In zcPtAPsUL4#aN|AA+YOKyOpQ_ERI96}Un~7V1*ln8d3z_*3~WG3ICkx$2+&B)T%krc zXLCNq`XvI}xG41q8=uE}4LjwV%5S$e@YrA&u_-Z$fL?@Ks)Ldg6SaL)ahRmpDGRNf`qv}HhF3T&&*bRKD>8Jfhfqr7FFnw#k$uC5@V=oIo%E&{J-Xu_qd z@Xic$=0ADzyK`OP9!yQQo&UY!XzLQu`!a)!D zI@YB`^AYW+4K4Z<7h&UBPkoKzuc)g62nnu7lC{_OOye7wB7)Vs%O~DLt%`yl5IbGiVyCx|#-L3Lc4{G$S1}{k_^OKiu*#tYdRdzF^IMBwQSA zMEj;gKpiGiZ2QXV&@W|?67e}h;{$sBcM1di5+C2dZx~4w6VB;QYQ|ei*ySUHrwvlt zZttDINKI?fc|nt;&2VB#z&=VB5V;i7E8bJ~h`w80eEr1xc&Mre6o=U#2~Lfdne+lGn{@mt;ro~Xfw;|OMx&`DYD@O;T8>x-8S{DCEhx5q#p|D%&|whp#*wEfnN z_hLD+hVjZU>*rY$%-lW!QYe)YM9$V+j<=V4{;>COx0HEt)yK-Th|1Oh{l2~(1DH5B z)NJ!_O|36QI+miM-crLHT*tILNBkzoTrDlzwfGj8J2%0g)3UZ`O!>A&uJbV_KlHqf zY0(!$B0M;Xg;`V4S4|I$ZVA|~>4YICmL6q4mMwfchihRLKL0*A-n87iyODSvXymP6cLLA$ufbq>K`>M`V@A7Jlbr^gR5YZ{3~O>-aoR ze|)a@=Q`K9uIpUq+~?iS^6@qEw6;;tvfJG(CLwxcsFPkHw*0V1YiR~eZLr!IruSyw za8fcZ5ET6sE2y|Z`c2rdbi^txW;V^&G)2lbYdyLj$s*xAp2+>8cJ`t@?1>BiwW9l% zo3loR^)qZe-doa^oL7-3!b9Baj`%?SBiuZTvE^ihxkiB&{uDEIx$?#L+V)#-*WNuY zD_zqVP?>D0VC{OOxcZ4&#jJDb6FOE$=qWp^)RQKWxVhB9+uky3H|mSf?_kPCiVjFaJEkqFPzxUOQ$wU=QP>1Gkx?E-+qGb1oZEO)-BFXtC z+!GufevjAfCHfVylg$mv&WOYX+*!9!=;QUf>bG*0M<;aVqvdphXPTaw_V5_OiZ`8-9x~SdK zQ=UIT@y&rod@Gb^wo3DBaVR}WiO;$*iKdkYjkalxW1Oe9J(J4%m)IYDX)QOH zIJ*0s-^tjqDuhwioXQA%bLMT#pw?-kSWziBjVy`8^{LTK=9Y7tE=-$OeQ=+jdrTSB z%gLDU$^w>z!yY%t8CDm2+~S^jpO7Dle!6HD9hCpM;NAM@Ejkfo$;U3=rwVo}42Vo5 zpVoAK5P2=>6)v?Fb@8)SCmw`iva#j03FGyP4L<8Fj}|B{wvcS>l4bDBt_vm+?}AB< zzwmzdIdnet=tG~PFnqL5etz=yhFL0dOZo6)#GS^3HDdYlPgavPL~UF?;@nm!sX~vIxrWHp?j<8UfaqEV|rn>KVWE zZn{)jHo9d$nV~?)BLeDYsGe3yjGeOKYUJ2BX&)IkOqF4lbg5H-`B_TO^Rd)a&ST#f z_*G)deBQo1GmoU$!Y8mnuBr7I(hjt-Z&uo`sLEdL{(7`>NPDZjI*KX6jxSDKBx7y0 zrCOyjG>3V@8!bGCd_`J3-TLQKA3;+?hFmUHX3@1D*OHOy#Qj>X;ZIF?LdfxYgfY^W zU;ED=`7p92a74By+uw@EKUk~fOfhAf*a~h^{`eJX=G-#Pj>R5A%D(4nyvq3Ri0h(e zsLlwR+T5B;4NlHedB|OLxzJA7R^{e`tJa4rJaQvwxi!99Y31k*%@GvVHY*I)+VF|F z0T&hbbGC6!H_kN*j;_k~SD-msnAc9Fw#x-mmFv`PVy}y_OCK40{@t`fr2*T(;lz;b zW7;S`rJ>Odsm5mX;+sNgJuhBFG~dyVb2;4?CO4#A5c=WzjnE5|)n??Pp+$U4Ua>~# zQz4=AUJ`WXd(kbQ_ue-NIpW*b>QoYO-Mh99Nu-B-+GP4+D`_~l zjk&pVO$G` zXKt>kMWwc?e4xdM3z(SXjlt?Ub~n>=N^EN#e)_BIO>A9~E|1f(m9JJ+C$Dj}suCy< zXy<)9aJ{vv@CrL~m|6nbm6}M6}E%8LYg!>?X_4#3{ZeH1p_ z{WNLi)t&aJ^+%$`Zlq=RxdiZ+HfszP?kUG~jK~HKSvWTH=+{U!_|`UWs^)#to-3FW zlhOScuUl}$l_Eudz2^~uZ*zrD=GCgLGY5YuS8wn>^*i`wnFIJ`8TryRcc=n>%nQ$!K1ZzNC36Nwd$slaml|>l`1&x2x1)57R`#4W!dvX!g|}le{XB zprkmaU?*p{dV{IL=1mG&R5dlcRmizvew8$UMVhc#$K@JjQf**cPl9T^BpXIo(UokT z6+8cxSGa9HKV-ccERN2ms9h=L5AsxNk~pbWr{6URvpXG#M@RqQf=JMh*5EqgP^Jtg zorMKve?5oi#OjT)#xg$iS3X~e7P5|LmMc)C6VW^U`0D9GPZHKK^;N&t<~3Y{Q9F9- z_0N@Q=s`Dq1;^*GzmGcB4lU?V$1kX1j^Qv7DbmNwd>8f?Ac`a~=&Ze27As$6_3%Az z8j_ukq}-N;1G(r;k}UP1+RLZifQx1+?(T*H(c;o~7?2r}{AMFY5foc_W>SSK&N3|SRp~Oq;EgI@x2S-1X@$O8bspQJ zsO_N8w2l^?SEL(g|gr<BScw5!-64fcM zmJFYpSI%MBoXtJCQsT?scT=rt^u1rF*zIx}KO2`fJ=R6#WYs3xy}CDkQlCA$Q5opV zT5U%SgOlZ0s0?h&rM5uS=E?R|gU zrta|lc|E$-To4x(^gFThDBF&Xyo`(@{_YPaAr6S|==5+A4#KZzQ@Y3=6PqzHUG>>O+Hq5ly(UM`y z*iRf{fOt+nA#oJk#S{umn;&I(;Kn=f%W-fon&M?C8D$L_xQ2p5ALsGv8685QoPY-> z`s@T=t5L$j?V4K4X{ZhCc?DzX&dn^$$@O~jlD9u0JTnx4be{3guR7W%*+WGzSUghY zL^;`=puciWK-JUIuKSbwc2a&@O~nc8zM}(2XS2`+zbd6(JiQ?+a@;B&rd;ugBJZ(z zJ9);8@xauZu|?ymWL3=cBvpH@^}P94rVT^9{mwLG^}`0$vjJj@#Et+E+2D0oI%=}dBy}tm9z9( zJ^c9aqL>(WmV7#1gNew)hn{^$S2e|Jx;czqzLFHN@Kb1T`g)?_CIe$}mD>$>e*vB3 z04D#gDK(#}y#5E_w7Qw=B*%L_ZJ$@DCJZh*sE(KilWh1tzL?X|*Q<#*-M22kb)FkH zXR3WIv$(7@s;z%nWJa3HhIZev`EU&^;ijsY*_4L~WM6cYJM|6`N zsTl7mUcKT^Pusrng736@()*kDdT6+E_IsxYOgVoxI>u6}*T-?HscN;t5ub8oDo8_viUX@D_63 z6iU)=62mkSa!Ctrhsn1H6_^~aSMid_pSUckdDlU$>Z~`R#oDubZ8OAOdIhB9hB%rH zr|!gOwrf6gEW2aeEbvqB?p2Bu7Fl?-u!r6;Jfbs=FH7(1UEv?Sz;;rM-sR~Vo0-rH z(wK(L{@4ZTpO{Q|m1mn|31<~v2TG;}SA-ekR+R*mqSvZ_$IQpGW(tiP71}s-)md_m z#1C&WG&B|gG_(sSPc3yBDHV0;7=ur-Dc0|+<}w5co@6B43f)GBYfB%&s)BLdX=33K z-P}%2@XRXza6#CHt?;dM&&j%QY9b>>HEM>-)~Dcu1a9r!X&Re-#i^A01bTtKz zzPhtrCm$<9>VCy=s`WiTveR&MLD=#Hf-GZI`JCC0vrh!7Mk0<)8a0==D=l!|E%+W+ zvEE3>V6f#Oo_|uiUCLE}Cvl1^F+x+9`M&p9J}1T4XyK4zgixke9$+#07 ztGF71oO@NFvI1p`&BmerMR$b!lJg{g7)#qs_UD+7^!r4skHsA{;kVzL<{H2Hc0Lc z7noK!^>;FPM9!-Uu(Qqd*x9Q0#Kkh#S;sVboWEr=$Hs~C02d>Oh`iCU7W?J0Nrkm2 zM<#1uv)2Y*Hp%KX`7E4``M9s79>J?+Mc&TfGs48J&z7#}X>V;Z1?9y5n8BCvR99Iu zd?~_Vd@u19alm61wbiNDoy`vk(JIhA#4ia6G+{Fm@V>!c&mcnlttfHgEL(fiHJQvJD7fZ2_ z{e*cx*eY-}h~Ux0#;conuNRIM#VxzPie`H{ojaN=S$sZVWqjm0D{qM^zF~CziwBuj z#mzY64IKnxnMo14KKBqC7bAI}S9*zDW3cwL39sdg+&KC52J1_{F49CMz5Ke8ub-AK z<4P}j4H4C_uw3c$NjRm#phSjHY)M~y;Ka@vz}qw}V8;2a5npUoA&ffpV=+_qcu31T z=W>zI#s($Hcx`7LOUhjR;ljeZe&F3tUbOOBz z+KI8<&I`_)YNe_U;jF9keDAG%z07oWhPG)vW#D=KeR|sAkp$atIzdG`PDybh?N>Zw zvrWDlYKt30;(|GuESQ=y7}tw%6RSvnCce5i_R*|_(E0PSVOSuo|AeFjUA$~Q0nTI1 zPV8$*ac)@G`~~i$WoQpOye<5cl+bd~kYB3=NheykJ|aJ3t<&LJ!z0T{lYqrvQaM5> z^twaNB<DedIS5E0gO3%5dGH3HhyF1 z9eYG{rvOzuQ6@__i~LWEqNHJj`mwW-xDGR;9;eC=@s55T zsZrNXFK4qJ3rxJ+L&B?S37c+_zW)7{*!J_{q~NIsiX@Rq%pb!v&XG^{H4nOIfYP zdFfAIc;0ckow{YGIDF1WErH&;(ADSdQ!F8k^P`0LrqS5(fstQj=_0F(ys<`p8jQz} zPUB~d4Z^asP6sZ#aM@ne$W!njr0O4-4b!+2c+*9cg#2YklUK@g85fomr@Y7UjR>=& z-+w;Y_FEd%47q5$$*+K!emh)!(B2@^qRA}u*21(&EX|on_S{K1KJ+a7#2J3$bH^|F zUy$FJN^jo(R zBnZC;Nj?epR#*Ktcm#J7Q`)_%FKg80Q!=R`)@8Dby*9d9WVzip%F@1@PSe}sGGN>` z^KkPyl0)zi^Hp95%tGm2&l7u@rSGu`S~tRaV@Gcbw~A3d_&80MX1St$Qjv7>)}-np zl`hwbxbM#;Zhw-HxQ7w7dS!8l-1AK~!NyOur#Uf|xMwd>My4rU;iBkHecrX)-FKzF zohPB0>jU~7?^XsJmZF5~QfO7;K|FYx`r|@kO@$2v0hEr&=%Bd;>u0CCC5!1~OYid)SLg%un1OM% zxwwZ<@m^Wke4}wJP`h#cVv`$0I)r){iIaz8(HyaZg3INSBh3jZI?yQ5XS2?fd!}hKu6k7a}ZN z>3A55aEI$Zg)flxYgygVl;>|d*WveC@EcFGJSxun_)4FM^G0klCj<7=-qAN*UK z-}B^zQWhKpE+((y2e{R|>1wQ&R?qm2*;Ws#?{Y$fE_|mtyF%5bBy|iz+9|J#HnrgC z`$<1K=I3YGzHa=aC7~9*Gpxz;&SQ&yG1KugY#SE~w8)?2K7C)G^fF0yn_b`I^}Y4d zLGY^+Z5n&P^y^*mizUQM{|V zx~&~%gcTTXgc+^u@r}#aCGT2(a+J)5+O?<1`;7L8x3-^QIFc~*Nd5(bP(k==M(sAu zafR{AzA@x+AL8dLV{X?Aa9TPWezH``!(dZTt(tu-H-kyOsN3X+H0p2i{2g` zT({-(`5yU7hS1fY<0w;`SJ8^Al?uySqEBObB$Kb7R(*W>ct$dRaqE;ICd+r;WtG%< zvubg?6Jc?SCOJIp-frdJXmfw|oU^oxVRIjUqu|gj%@>^7&#-biv(=UKxD@H6*3{6K z(?j&+BQ`FYdRm`yf+cihtxjsQoRYAu%T-RQyjRvTB8??7L&|~UHF%s{-kS6_9B=FX zobuGbHL|rM&B@{#qu(CYk>BL>%ux=}qF!$0ihi#3B){J06j!8gEDMHl zkK%1|GyPbYrQEb6ZJ?ekSE60$O&m!j3GpC?MNV>GEi+$EFOMtT2xmM?XGqvyDT%A% zlknF6WfF~-i-0)Vm{VeI>}42f4_8Y!Zox53ww&ol$0{lYE}WMZbFJ0W5p1s^DMTbc z2@dF*P?(k;w2<^Bz+ki2qb)xAwwax|LaN2T#6h$^K9g~*MV>^#yn8Syk}-Xhm&a1V zE1&=$mRHaH<;i1{!&*AK6k$#2uV~J)=6rs7{j|!cim0Y8=mo97P%cBrP zvw9>Sgs6M+rJIDfbWtU-O-$#--3m{vu##^*kGgCOu3sA04(`Eo^YY`|q7I@wK3a#R zDEZ^XQ5X!|LOIM3hPgg#%QXpw-af6rrGGRAhA61 zQ2Z1&b~H7eQR{8d-Yofy?{``t3 z8-^KpmZwgaXBlwa!f+(!Qb~O2N87|3NpNX+OOULT=;*5tt5RQWgC!&4vZCLw6v;aT zRblqT(w{NRY=05si6=i+wubS+&nPlNsQ*^YMkJ>inNP%L^*9|kNBc7ouE~wsu2xG% zUL4Q+*ch_MG(<1Yd?0dRe+yeSvAQ=Tv6bSd>Zh`WE_S`Z(n~3femhTidxnDEx9agw zCc$DS_Yt?j1frxui&SZn{C7{L^{=|<;}i6&9b+BMPsu9FA-StKLv^>-Nb7pynTzhe zFBMA%FUUNj^&*7db|XToXP+g*})r!9nO;=O!7Tw9yKgnx~aL#}1$m_Tkba!AxUaoPbuIBFB<{mjpnR()$K11D$g*K zT*={NX;*7y5JSU7IO3f3)q_+8oNB4rHd1`UH4PDUK}VP212 z>U=lSsfcwXfvSb}6J2`?of{UW(sa+qNY5dRo>yPRK4-gRMh5rm3f%c}hTv4OS8V&$ z$!df~AZ_{ab1H-WiFy%)HN2eCzS(A*w7Ay8F@ogRD#qH*C^|1sH;&BLHP~J%eJG?t z8W%y?O|t0KEjSbI@^-8Ly~xFaFuu-V$s)8`D|S&#f5eyTlOKeYMnsmVj(U2BV7&<= zBzl8*C+0*wYeMyu*5O$i<DQ}rB7~hM(9j(3^SpjL zVS<&zmh8x_J3%BFkGzz+VcJft5l54&(bveMC^21hgzZBmIv*mDzd`WI_J@~Bc^og_ zT`H0;PAa=ZkGx9>TVm0uwl1g`wRi1nr|O)Nt}jb^F>I`4Q}rnmhrIC)!8~rk)257* z^>N5}vaYYZDqH8mg9j!w2b&+bx}VLSjrlfWe2&R-h5CY%|M26&YbwX^r45QCUX4vi z(r1VU82K-j9b@x_HU6w~(zyRJsyVyIR6r=!p-4HJuSCx#F15|5bi8Akqf~2rpx0hI zj9lTE*)=Mz|HBg!M-y!zrC6k-6 z&$D;viCM}OEUuY&$AES`J|2%;*~_eZj`zF5v^f6J3(N2_0XxsTPW`c3C(PY?Kjttt zao@=4>l+jf>%7ye$u%4ZPr7eaA~E6Wi^ec1b*W|PQoqb2;23EyCQPqI<5YTDPYFDT zmUH}e!d-t9?(S_Y-^rxeL=#uqzSJ!LB>n8%T{^4dRp`f+TlvmRDqNo|l=87>Fp`Th z$3Lo)B7XMtefGTfPm7*(>cC`;?liwb^H+*1cAeu60feeq-MgP+Lu-k2J6c(puLbUtMF&vQBX| z&(xoecKHl*PMKh;q2F?i&qvwPlv1AmCVwuG%sz$D6i0WJ)4tQcc0!iG5A!{N zoR7deo0l7JOHK!_5ZvUf2ztnB(l#^NAEXl;+Q zQPnDEa^7x`KfU1`JzU%fPn(nTqNg0;le=vQqh`%_Qwwjvw8+Q|K}Jw&DZeqv#m0`f zh1SK+!E*Zbu!Ay&s7j?*IOZUYs`XN>64jbNP5lghb?8^Efc}_bUO9BLPr(DYhH1Qg zEigpu3K13qp2U(qsW5HZV}F6w$T)F|DY3;|(QrOSQv(m@aA{rE%8pLMZ#@hNRPI`C z_eiWCkJ6~d6&~sM-Z?$`GK^B(-0LMv06WXc%HSfNQNIgUn-N$< znobjWD^iRaPYT}I5;945I@@k}J{(w1ommQt)>Q7nI`U!2KiaWY68kHBaK&9zCh53j zOxk2QDRr3fnK#I@U(`=nvJl<<+2eIlCm={;$XrW(+9Qetr#_qXx-87d5Pm8;Vrh!Z zFu0JhMbo_8rGMe(xodAlMCNM)Qac-`SU6NdD-%y_R(Bm$CXc-&czNNme)w?4W$chH8)F(XEsV_7IGjXq2aSU+ z>?s#fS(Rw!BXaKx{0ky-c=7$oWG8==sj5FuW<|$*GnGx$N!WS93a-wvj2;t;`VZ@F1Ag`qKk z^K<)a;2rl{(ySVp_7%;XEg!pF;=6ownN1vzCF48jKf}o(XIFTR<^S%T0Oo@UX3n*i za}{~8mqR~axK)2UQBdW(s{MRC)K1+cK>b8h4xXtFL&_q;DT=;EuBw#l*bR0%B}Y}f zMjwj%K2pQty0}rZEn*s`J@fYs+6^0pKdNoFX1jQ4o;>$4&aHTe+Nn7@J+%ffPx{S$ zmh-$hvL3_aYjRoKcws5AnwKR`TpRrsN$!Gf(rV9inWO@5Q*RqJ$YY5KM13Y2!Y^o7~Uq|KLa&>SbK0$&h-sxWfJY zBmFb*aEq%m$XHGYGOumk5iCoT;l9%6iiWEEPXPuC52{p#02YM zbxc>k@)xl)+IrcaOafp0pj)MMGN06$5y7<4HF!#(H#+y6Dg-$loKS#Pw&^{vG1q9- zum7><^{CM{fvUn0bW#jbP_hPhlB3bdgFk5CX4{=cyZ3}1NCfovKQ9Y30{1mrxfmIN zpCKH68`EQ{8*5-QH3HlUO$)tXuOSf<;J^O+ZQzzs3&3XhA8)&@fSug~xC7;Q(a>b} zUj?2>ggb!zpKtqhd!aK2xR8;LlZTt1iyvIZ%k%3p^@F!C9{vEn!r}9#-~%3j4^TTG zvH0`ek5oP$O3J0WkTztHMQq#o=W@=;P?7GWX0<$+W zG6w?#?f`azUw4H&yEyM@2HZi83bsUTD(+PP=sK`Nh_h(FAraLeMMp_7yj(ljcy`&T z_IYp20Vq@eq}EAzLJVgfplj|+2Tvq|nGj_F+<;I~fRAUFw9O7~X97519pO%PR?f~= z4)%xWG!74sAbC=$50F?3Iz&sg-;f|Yql^GzoaX1k%@`_0-g|;5ce)K(eBKF$k0$3pD`L6 zh1WoQDR5uVHUIk!iGX}BiUz_DI4^njCNkje+LgOAi%ov-#o``Ju!;ha!k`=Oiv~|5 zf*jf^q*@k6uJ%@7uGoXSvv+2Vl@r_!+^GC}Br{#MR?y^i2OVN0 z-)~67?B5^}1eAk&R9_IwA~bIS0~H{F-T#(9-E4o@dJ(ZNxgqBg3cA8RDDXrgMo*!t zA#Oe{KrY9<%a;xgOhA(ca3=p=AvYD~k1PN|L%=w+4+yRYH}vj*4!EP-!_x@v0qp%> z({L}Q@$ztT3Gwjm3EuY{ZvUGG_3+2%o&Tf;Vb}&34jLybxG5MV419n&5D=;F%?&%# zJ=!%?jB}e2KOr7Srv}nZ4`?8S>My15Iu1)X%oY$~0Pg`@{0=MdPN;)G{U=}jVRg_{ z4O~(HU~YrC3f-5!-;jvZ(|-X7L^-ZqB5g?W4#y7UEdFt*Ms^l|AxK|m)krCt+fQbKg;=i*C%*Ay_9Xk#by3zgj_8I>rZ=wN6=K+2bx@djBArVYx zP(~mG(bw?uL1~YXiIoLJpt-};wCh9nTsf2t{Q*oUo%u4P3Ukm8`Y|h7RKWl$cV2G3 zJ<5@PX6YX1$Uhg||6VOAy2L6o0L416lY`-elMZDhyZ{L|Pz%IX^ot@E49=hP!WkWt zbR8f%`TwP*A!kt)&kY)&0^>x1^Wg9dIn{BK#sPdi?*IFGRR)yB19!p8%L&q*9iCi! zS6f@aP~u|c;`v9PS^nwPZAT#f2Iyv>?>8iZoe^d6U~&SM!QBaZz*j;(YmffnpSk)6 z!@S*@r3P_WD*}Za%>QVaP(;KJ)+Uf`@)()IoK0b7V5Yd5SpbsRT_U|d0AL%9q=9@^ zCpg}B5P%>vssKRr;s*4nyDB;0L{YHF^C~oL-9Q650EFfKfPg0wvB-id01!ZV`M3dx z=OGP1>A9_ggEQRzm#yyP#n#RahX`v98>$TLu%-b;-YbyR@ike_>Zn*QTI z_v7sUW(qkBX35S;3i4Vepda*JwC7O-BLKN_pr+kG2gMcW^f6$Swso}w%0V^4u|2K9 zk^uuHc666;3IN(^NW}8rAmIVLLc7@xB-tKj2X}UanSx}@#lhjf2pfDKiJfK?N)n7yRf*~QGt0aC;7oj1*GiXw>40S>zHj>O$RNJK9?%5nw4 zI0)?V&;JUDjQ2d>--wwqkQoj?Y@V-h=7 zi2|N!Nr8+f0K^y24?1;UZWQqVo;*%L9v;5EH42F3rV!u#P9DBDMa+SIP(^eTm-J*N zK*Izyqzs<>4T(tRK@|}wHvyJVQ2TMUhcskn^s6BC`_a*lZ_9=XKhB^-1kd{oi6{ca zuV41|n`Hs0IPq}t1CP2}ciM@vdsA~~i2gWBm{OEr`w(>Y<%1^@5ykfx5QKn<@a`^M zj6hH^1LFZjSjc0bU=?^*wQV3@35Aq{dT`)J83aFwszSW~0Fgung`d^R4C3&pVghC~brqAC|~egd-8Jul@9 zhdBYhtDUb$LG@&-g?&9hs4IkTrAP=BD1Ipj6J?<8COdaga!4d!k04wbL z!)pM5AOHZ{|9bF5B7#Lw1OO5$z)lQtXF@sle#ri}bKcAQQGip)Jl<;z#9M%Vu=u$p ziZUD?5Wqpw|7$H~Wx78pes2t}pB6tsAq(=15FhG(Ln2x(qAVN;=HvtkCZr2fC%74u zZo_PUO-Ga#$Ip&6e;5R_AqOmu4?4vZF;szYL5TxsJ3#hnT2UDRxWMIb@Z~%}{=pHL zmH129yH(bG*}Fx8-;aPj#c^e58bDqM8j1aeM0~k~vT(qq#>2<6o4xH-n!#!Z@`ron zL{L`SS%Ur^izbHiyDFfB2`~r;3pT})e}P31Bu>0Wmw+`mIDvAWGvKL0WnOZecl+@b zd_E7Dmj@G;5$V5?4%WUv6A%j)l*w zFGB!ZPQO=ru)p2}H2Bb3;-CST$e}79@IZ2d5-vBSbXuTvn7!2v7*xFmRzU0GK)W;c zw12M3%%dHrEC+gs0S0g|Eb_>sh=rSz7x2dlf@BGD2m74adrW^Q3`cQau=i7-imv}R z9L6A*M=3l&5O}%4_S6o9U6wSI!}6xrXRbmZpZ)*I_XHI)6y*y+%ZI&6va1y+54yMj zJ3z@Cz2JCo8z8>|bQtLS4T*TDh^l-Lp1AqIs((7kQwJBQ1_Ao)UH7!>kah=Uw}hZM`|bQzMo{%({~w_3o;~aM!+5X~*gql$;~@{6VN3I?NE@Ls5hR?FjOLT_?y%?N$oG-yQnI{Z#}?X^S9-ykRmJ4h^tR zZ36J@HzdLltZh)l0xFh#{Cmk8n4yjiP#ybM?1r`>-JtU2&OG^pd5~F=Ww-$l?Vumj zO%#|rDB=;|0yYbdGC;c7+5d9-mj`p$Qxasczc(2r3hxw3pc{yp{$N@appP<^U1Qlf z*8sNzTj`)+=ME|WR%RZ*7Y?r}_V5Bq7zEu8^!EIH88I(3rg9FxMd|;D*mjHKX?|SzniUX*V zn1c9Y1O=f#g_@5ihLu+Uke|S{9jw&I8KVdU(C~s2Df>$RGq|G@+!VMrGpJl+Wo`xj z8_J1xHvCaZ4V|BC3qwY<4f?_0R|5tPCDlNYPl$_~2dor#_9J$b^J`NER9K-^>+c7M zdA4=uF6d zLn8da{wj(%1VAw41Eu&Kzv~7jG?btlfkAO}H32sQKwEflCws6x0i`LZ)}4*WEq$P% z1QlKZlwS>r@CJs2Dl*{o`1ib?^Fa;$N0|QokR3}4HoXmC1c5nqaL5b*Mic>o15IGt zWOqgRZ~NUfUn_evxW}KSlpz&%5L9b}hV5WJZ(xop4sd*klZS^F3YFHb&QJ{)B(nc@ zIlouMgT~m$dO!`#Scl6!0-)l7nkoPUUQmVJvzpzrRTdyJp;YQiXRfUh1OP6BOzxn6 zI|5XLA^;vPPOz)D3t&Hc{Z zohjo0Y;~tT{`*n6Cjzg8R`}57|G|Vh6J&WPV&UZkHE2+TMLD4!Gq(u@tA?FrHk8Wm zHzeX3a44us<>Ck1%e#vSM=M7-IQRpd;ra`c?3%!8DhIM*z$y;rvS&f6fHDk0J|14K z|Jc&qIfAgi@cl!MvBaM4woR0;C_vGH*rR_QWs;u>5&O^unfB8zAnHMxG#5<ypz~(=< zz+Rl$Em!^9nW5mzR`79)C;Rw%`#J-x z|9ta*=9uH!`&{R$i!Kry8wL4612&c}rZzUV`VLO^9-jK99xhIX&K%sF{G9y4oGyk= zDEn=DZu4*Bxo03@$^oYCbZX+nipTV`6_rKWNi4}lrt%B}le)zcbWdLUKbd3fz?7Qb zox5!Umqc#*cG)HjypN@x^Ey6zqsE(=Mw^VWjzqC z>b<_P!Ixqw{NNB_G%d+$Ea$c=pjWQzQJ`1+Te+0~qF(qha z-EP1J%3tmoiw9ssF0k1}G@P^)dIO=R zoF-!ygh;khQWj75ZcO$l7x#4==X-kKK?L2Sq1j7tkCVe*e^Sk`EkQ{tGUncuo6%mF z(rB!E^ok^BW$5&DZ_mfI^Mj4q9m1^4EV|4#Q3-K~acijG7MXurv41!B_HanRga!M=(0#@S}G{RO_CkVpOQP5q`6KJ{&998%t*1 z_sZki3!6>U$MfKa&&?;20k3(EjQg_Gfsf zN7%3TPtVOT3B2i%zNc$~+x^|ng~PN^t4Lcfo^VsMN6UQ2K`tZ{foPeyMemq&-*fv(01*&D=C9VR4n}w>;-xzY6B1XDY>)vd1uyBX%X$;S%)R5?Ot6&s&baQE1ibHPd=c5t;l-UM zXn|w5%dmqqEYIt!7F|4`2>0N7!W9h(^j|Q5@}V$8FrPjRdnzC*Rc&OCsmkV?1z)t_ zmsR_~9tA>qlmZ>h`f?)bi23ES_gk&VkI^ZK<9$8c&@(?j?TrmEYtDa)L4(ytJ-T`{ zD*eLvKt6lsS|YPDy0lXt#i7PC(#3S?y!qlp|iGB|0l&o~Z zhVDFN!%f)u*7ZbiD#Nndux6RQ_iC^GPR-Ov#f;mopG)y4h0|jD(FL8FbLX5FoF&5t zgr9Y$)~?P4$I@pz4!4NLHXM)Nktg3p>8Eicw$Z6q;^ut_=z0|S;9^c&{cYM!e+m|V zq0WyOy^Pgfn5inqTTmN}WIvs`P*HHxwvk&qmiVk0X!5Ooo!P$}!vu9dx6(b@+(b&M zXnVH8(MXvgne#W3^-JwylSV^w%xC>OJQL&tFlw>cZRy}Dr~BCQO^CjJ#%FUf`wwzf zF^&tHN>u7%9SbyTcSop)x!BEz&&pJApTbmx7{rqQ@&PvX_(fYmC{y%t-XQaYh^pT--ijak)`K#w3+9fO+2G znO(8y_IdN&rr$KbH)(&%;oq~xt-u2P0pP7$lduJ_+R)2^?Yg=Bs1SYG8D_-$HEb6| zdamQEA=Y6&DgT<9hN-;}boJO;3rY^BsPu;Q)D2<9HM461D;5!myPNEdy0aOi-XfvM z(dPs|tfiHbHd8NKFQewUMQWK8lxD zCSt8K0?zZ+RuNxskBgdQ zt%WkGFL{;1T<$tU-mm&Ve(2r1#xGyB(CU->UoNeFdMvFFtN(1}seL4B8ZT20P7S@q z&ll|!Qk(VK5Bpr^lTlXQ2vz$Y!Ga0DNZ~dLhEzSFoK+(zFg9&r)kAxSx*w5=<<^`y z>UVcfLE!mvnCs}VQc#g-YF|IP>l_VgkXGNWWdlhyBgTh34O)t%4(q#4)Tb-5OZ8-ayM{S!1qO?GT*$p2@-TRZ^*g%k*)2a zU=z}*Zua|kLK`m0_h-HpZyy)L-VhV-zA_z~x8%NE?%HZ=Zxo`UWo`eGA~xj~Nb4t) zRM7B%-X~fjy#9p)4uQUx-ZyhDVNnI@4)-{!FIWz!$~Xj1JzvmCt&BYnr+$#ctr6WV z(vZKcMkq4A{KBo6Dm?jK5}YC?*--Zef<0V9h}ws$8JwH==F`6tUQnQ|KoV-`+Chj|97Ci4=3jZzKi8VcX z&3Lci`qt1&No9rhH&$|uE6RtUlE*%$)~Z`1N>O*7havqRNwAB=UgjBmj6n&IynODe z{Ubd7i@zzp>Yuro174S>qwJb>tREjkZJF_f#Varwv{N(IO)s%NP!tC1$@nm8=Oz(h zFKPv1;Q5-%Ka-X=ODZOLsHEZ%cE}i;5+hnqHj(o3L#_>eL7nMn7v843kuC-zJg!`W z9wkI7{7d=!bMb2OqB*l_is-0vofm?m#F1^X+*uD2SaiAaoqOs(pyZRlMjuJ^WqFf+ z$~@*3?HbkS%N7c^O`)t7&+qzzKhPW6X+t}0FB!P0mi#ql&i7b0Y$213h2dfHdFHO{ zEPQ%b#nb6i6I6J@yz@?;oPG0+qHH7!zCZdO@21$Swa_<;M7tg@G|y>cRaV|P;i@w| zB;<4ni>fjxDUc3zR}I1_HTEW!e|&$6_!KdQ(oysCB697SW6J0HH55LQ9fp>&n0M_s zw?opzj+2TwmQr3Oi6;J{?Sl>&=SEszaLXQ_InStt{1qD~ZBQ(`3eiCP=2KhXp3NZ0 z-gi({_~!DskBnsJ8*_h}u}Pq0$@AiB8ZLUmzZv75kral3DAuo8*UPq~Gv0Au#FeU( zs{9}U7}|+S>fzN@Z!XI7#!sgk1uV`z6>JmSms+!MHLPTH12W(7ek_S#{cKy3QQPvR z=ZoMyR|{L6UD)531$pA7)gP7nrE-^Aah{4VQ*QhP{b=PQVj##N@#ne9CH)}nv%*qa zDWx>Bo$?`C6&uu|m4kYV%+(_fi--IGHrT_y8R~nT%r@14Mr^AY9^Y^%guR#{@!aOi@`4IlyGesJ+TmaKh0%x0OPSe-(Uxc1 zbbZ~BZ$c{TaMkT4)CX*CUY7Fh?1P;2EtPM7F?r7I<9~I>)N;yu&$D$%v);t3Agj=r z!;-}*sQg^Hn3v-mu#d#=3~(r&>l((qPG`xcpA=47tVOf?x~~^=IN0*3t^(V_cN3UrrA0=c*?MhK?%d&$zv02^&Jb3 zayRxTpC89SUIuftp%J^edbtF}F6q*)anjM~(b3QnSVNw<-|3#4&Z7Q)_3A=*+>8yX zw50WMbYGz|EUMOSTR}}V_x4wm3=1^+Y(B@uxeeC&NhY$LA^)|$z^^BQ%5U`9sj{H( z9q)ZE_zy$2wDU*_ZEkl63E?o?Z_|R%HFy1<29lCCSFOCjhSGL^lF)Ckfk>rKgLS9f z0={c&nZw03So#l?w#w1pz)~h`v-$BWuN6~wq;5WCn66mg{Txck#ndv}yGGM5Lf>g0 z&%QjnJcH`ZWkXzS`|zMc)DIkiE2>q+H{Q_1hA-)0Ng#!m%1sKR?$n&^n6+kSJ%k6dO_5o$jFJnM`-m8bP{ zmpWoeReX4Snz$*I{P$GK3+cm;=I@58Kc0;_|J;A@rW2FcfC=_i^D8auO;VnNyZdYj zYSh0ac&0@-!2WMF#=0&en*-9%p#{tC*%Fh^%qV+8s=4 z6a3@4y$JWJmk99lEDz-IRry>c2;fL7BFr61o9?I1z`H8VO~}#Z86391bIX-p2k`ED z*ag77&qFwX2F+WOW;%M<;*P@F1hhx1jJlM&&0BhFCY&$thzFL?=rA{&S|3QPzKbhy zpcOKWQ!+7yvzkt?yF|-h$IkAmqEj85c>tpO z2|;CXW6yW0?{0gl-*DC?Mt;s7Zxj0broM4KOB-cD>86r+KM_HPMKgdvXU$ClS)uoG zzh8X)WRD}Iml&sk0M=d&z+DEHcUZp&rJNywX)9XUGTS3E&u4}S z9!7RNLYK_@lwa#`d~y2kFi(JFTSedA$(N0e!LSWl+J1n zvXD2TI)~7Jdk;9o_5{|Ct(V5ta;Ni@`wzkS!D#&%z^T&gfn1q_*pc>4kiH58)}ViI zKX4&h_?hctKm}{RDqqI)R6}8`wDm(s#O8dQa@alTgl7wqEcK{Sz+ z69uz#ni(Zx?3vk?byep!ML(-t{Ialu3kRthWwe^D++DS96XL*?=?yf00}S^AuBqqX zD19@h4=unr{l+bUW%)A)!oCu&|L_v$20ER11VnUw!BxRNB;y?X+J`7ua9xxO+)a5zuFL4%IV{zJ+z{ zU+8Hokqyn2fUqN_Xk{FLlBf9YVjCzIcQUId1-;pq)!^-ttLPvTz_zel z-rE8s29;5u{|<5?vWSp61I$K%qYTzph(gj)(Jjdj^h1Hd2@nC(o7d3U&t%{Te~3)C z<}RK)u%9W{AZg126S@e8QX)|g`o0OFtiJ_3SmBav0h}kkUnt_J0(G(%BAF~j#u+o9 z_lWm#S8Q*duyRj_)!#5M$RbNBgZEqo!#_jtT6bx^PrF~uG90|3F;A^{2M{U>t zH<4K4pTP7T6_V-dWVBJJr>(nK`QQ02_faHV#kluCcJ9w zn4d~DA44<$o_Dsu9)DP5WQ1sFyU}ldR&Ay9hf}2de1*%#Kv0eutK$JEzlTnD1FTH+ zRvfbc|C&EKcM!;<|Hk+R1pESqTEie4A4oR0J{M{4zeruu|8;awVlIcK$+ySw}gbxl~ zMF9GYnf5FDT~IuNu2|}Eyp_>14bl3t6NVoh!3G^06QE`&E&5*Hrt_7Ea{~4C>dI``@?`*6IyxWR*VvweKKee9v-%~ya6>gUxz`@ z_2Y+4Dc{$2x+`2jk5bXt-9b#`qL8#mPg_%hX-1mhGOI5v+#C5E>v0%Yxj@-=8S5`s5I3T)#CCv<4v z{XLPck>t1YYp15HL=vtn*#663oL|(x)islt=9j^mo<1|pME!-4;OfQV{C;YOr7%_9 z^@n+Gl4^ql@qX6aVL6R>lFSH`L0|IXt(8FoF}4Qgb+2LZ2B;fE4)EIA0NDEtgt$e` zprYGX&JiffAA}FhpChbAm!}5)%~>tDxMD)WP8jL^D^zB2V{G=<8N&kX--zhi&52dFn#Gf-03kO`QSd zXNNy@%ozwOHE8iu-xu9LHvw|j>Rw9^b10(y5nko*Y3SJ+@bU6o9ZtI`c{INcjXo_0?Y%U+A)tgs7U|8^fN4Obj=YE`bC!>T> zA?{zF0yKl#0zuF%*BQvV1dC5VrCZKToYVz&0BiFo_A;k?{EJIac$Bat&qF3*?^A)E zd9g(K;E|*que_VLx34;Vsgu5=kM-NAO6r!PCn%|gt`78B$f$WOeX?I=p!$9&iw2s z6`Bq470v64>}Kr8VSNzUfXa}rN5;`SrHA~hiT#I3K(Vio%NYOGoWo8(qgA|#f4g>3J2+OKu-G6W|uEhY3P_OlhqENUN@|wiaIJoiAyiki%0qN;K zrZx!<==yKLGLjzPoj_{A#Aiz*h0#U%M}M+4Y?Rs0A6Id+iII90Y3?!1^ca}&m{jws zK{Bfoo_>gY8z{=H{aVlSr6=6#tAqpAv&Up&SkTRN#ai)F4)2aC@6?(H7S zi;o)Z!OgUktClR3YHl-xCxstd(@dSm%DluBG{dA9TVx;0b%CFq$C@~l*Kep#^Rc=Z zJfU#+#ED?j1|jiB*08mW0ri{Hc=9W`#qB65&Za-JhWb~sl~Of>$;rWRNI>+|Elm3y zdLPpceA(|ub&|m}D(>XLzPso?{n!Z_BdhkxywiOplFghzu+W7;@f2t$y{n{A(s;Wm zjAO<(@BW22v>73nDE`SV96tNJu)FM^WwN7ss@+4cqgebs6_MjWqb&NMYoZZ+yoOxy z|2J+I)5x8Wve+3Q|42GQ{{GLixBUcpg@FhEjchweyp|jQg*RYg8T%Rcaz_xhcUI+)jfZ7>| zJ_ZdBUq?F5I8Lgq*26P(AKC<3y#9t3_A7?Km zt?iu%9(do*#cYhNeUC~;Yyc<6KozHgDVmo zp{(3)y$k{8%)gJ=UjA{Tt`BnQSsmfk-A2=}NJ05CQ+4l|lWr#d(qQmWai!7XMdka))&XIi5}+iN$v#38!b_D31-i!w7$QqwdCC5O5Sa# zirngs@u@9_NlBy^6M@AEIQYlb6>nhg$JU1K17?qi<+xRGlX?cnd<;aM7*O3nXsR@C z!NFC*9Yk#!c(THw;~+b%i4$IY15V+PoO?hU4G5a=hz;h)G%;GWHS!~BsBN!rx9sN<5mcOl^-dM9(m>#bo55)V>(fj(Mjn zw*~Ryzf(5XMM`z^%V9Ehq7UnHE88)w!P@u>W;aN%`FN_|_iY!rV9tTFoqotP#t*@5 z*t6dOVvUi!634b*OnbFGp9Sw^X*q?%XUKd;9ayj`}m^E>Fg=Lj1oPxS?6O4lW8vL`?@7S=7| z=*0kJnoJI(pl25PlYLR;2{(pEQn)SA^PmWxXR>i7SN%UfpZW?+cqJZ85@q?{w)D`Y zm_WB{nA?mOG?R`}i>kB{r6a&EX9L<754`qHL2tSg{{y`541|i5945Y4nKVyyIPmNe z;hIRCx`d+70_&W6Xm$*o+1#JG=!e%s!<}46&PFW$DU*#~sovq{oay2t)1sfcI{Ff` z3)(OBXGsq4)ZyK_^yvP!PGd{jV zItW2hp$fpT0rsf3;Q+~IX(<77m(IsJo;oKs#-a3G`PX)oqk{@(k~-2~BBnTP9{cK}ABLcoJm|ysyAj%ISr}3*vU4TBASF zlif67Px!t~D!o^E>p9z#JHgu-O>af|KnSOX?_4cZ==t{&e23P6h5=u#sh?&JYh-s> za`e-Mmb(^m!s+1 zw~FO98cc-OXQc~XsOU}yu0F>NPUjLd;)CJTzFywLk3fP!gI?rOnVsInj!2h3s+PJrO zM^dWk3z*<2n2b6j&-E}?t|*aL|B;py2&Y2!Z7u*;9yr>&<%%r=P$a#9{Fyzvj%d#48F+J{XkO~&*UwJ#-Pb)Pb5i0Z=a&gZUhr-zvao`P}SDEKYy;;}qxafyD zixjAr{YVzSvT#V-Wm1{kDEK+Wh)e2xLQjH1e<*x2Kyv&ZBTDd(1%B78hEh8~s|bKWe>>AtPN^7l9PMEDSK&Bp0mj>p#xy6CNF zQ+IZ~ZWDRTF3TYJr8jbiIJ4K^tfJ<1Sl@6!t^`TxYo|>ap+}g#iCsLm?4h=#8N;OF zh6I^>?!n0^5|C@UW@1^=BrR!~x3nUBgCBIg_1@*vidm!jYMVWK!)P~~vYOT}O~@aR z@I+sSO3jVJeP-CB(@y5beKOd0ngwWaY>8Xm+dJf3K;PFQjS`YRI0RpjpgUhk)XhmAs*I` ze?u59gVkC9sp-6f@z_{){T1vM98)2#9BS_6zo2AFpnyK1Y?1oRX-lCt+9&ZgsT5~x zaoEjptHnMVSOcisI@mh{dv3#rInxKLZ>r+>cJr8dnp}8`I$wvO|GRaV-gN-x*N*5$ zNFg^Yk_nyIIw?Do8ZZU69%Y|~>ZfDZBA^9G)A>f9RuLSPM}VLw=5BXfn6-pLO9Wi> zf(7{Cmea!e27>p`5RbbD$nCRv4N(Np+@n_X{we2sg%hckd&>qWhs)g>L3gtlBA7=Z z<32d3F+Mw+`?0(Fo??IY0oKsx!?EOaX90Z)!**W5_5~ZH&l2IxvDKQK?N{#S9nwL% zaIt+DMf$+cn~5c7@uC)S_2FI__7AhBKjkB)k7!f4!s(@aWq}OL z=-Y(&EVlU6+5X zK~B5t{guE8X2C$t%Ex{4q4!GM}1B)X$6S)CW{1?o1 zdX{8W2bfGB zh_#B9rU%=R{g_nkm5%5a+hVwR8qLtO{hJPDvwD)nI|h{@q(h5tsQd%`*nk#ufEV{q zT(EQML74+EW?cm>bBG$EJ7tJ1f^Jh`8H3BamXMm;m2AUk`83Y-ZWg^Peb5K}oM}ws z80p?8pFzIT;^VL1We02mDx~j@`|`<;Pd9t4k!09}I~}>cf3E zN9-1>mP;-MB9F?rQFzr$o8i2P+Xtg%EDbi#JPmZcc8m>J-^UAQHn-*mGpW9NDOPIp z7LR`4Q|mZ`1l67NJm(SB@KT5!2H1?Il~4cw8v<4~$_$!S&}zJr>ZSr=XwRfCcCSj~ zS^Lh%nA)zWaEH};Qi@ns_E!=2@hZP36_p3DGNOVlL+9#j;ZfWW%b1ks zkAc9wKZPpGv2F61Nk1HH0+g7aTO<7cGPor^v7p#ZEv987uBLo_o$G?%E_{t47yfA+ zT@uqxHTef)X14)R%V1m{*E_b^uq}$?S1Ha9a)-h~}t5hAYu@@RF zo!WD)cF;#J?k|^)c#Gt0_1RtjAw_N_ws-IxDa>!O z!;+yvDyRK^+2eXhz(GOa9^K)+-Ydtnu+<0b**m!oM{FZMovt*+CQkQVM^mjx2*mAA ztyHUBN?c94_PaGdtUdXW^i?!^&cEmY5L|(UBa1s=JPU?hH^6}#(lL}IcZrWQV9m9r z0!j0rv{UaR7~mHjzlZ40gY1}la2q6;xw$L(=DKXG_sHXqj}`25HJLIp;HHe0vFD`F z5tIDV-NQ11TT@bQM-;QrWudnF@|=CL&sgx?o-+2D!3+h@c%ucG$Nkyx@mC1LC;Wkl zYd=nj;m<$SpEuP_7v36ee1BK#gTMK%Mqb%8T73bPl#ki4=S zC#@09iO3-qi6HVOdWcRlT;7it$7>zuV7(62{ugu{u$@9^km1>@Q+-qT(Vq-WWf=h?9pS3w(C@(hriIDRsN(A7~lN>vkwD%x5}}7pv8I(pg%@b$~{8;x@QYd zZbq|0HSqoqzt$u#3vXk&Yd3-ZBS2=Lw;@~D)U*jy;bA^t;?G6)%U|FM$!xBMI}M=o zia4ml=%an%r;IOtv0O@cowUW=&+z^un>Yxlz1s#{|6io~1+eW&N(!-Nh7H?8YbuSd7=f5fgFcb4_g{^ojX9gj2hHo>QpiNVC*6tqoYiDB z{r#-O)xvak>Nit5ZB>*0C!&%`cdbowl6m&WabbrSIPXiBexBNuDVt5GX=qnZzfker zCavsfQP4#17Rz0H9}@QC;626fpetuj&q5#J2-sD^?B6*PcVB+@2h!y}!c_G2+9gw$ zlE{D!Zwb)kFjvkpggzynqIBvpKyzaV-qEi;uutFr$KRX5o+}eLva75-fGDpcVioS_ zJpp9JA`_`)RYB=v9&k5(QbSu9+&F6h9GxaPt77Xq)Vq zh_#17x|k|r4;S}cmLA|l0?mh)zN`>RLYY-TmjeA7flVddFQ5)bd8^Aku8 z*8}d5H+WNmG>~q;2HSv#j3xOpktITkQdLDy0ZX6nAHyj`z~+-LK+Gmkehrdqpng1j zQjXehZSiLG?)9gE?`s6K=oZ?6-7H>Xe);KHqbCZh%<7&3Z$JX`(}B;%qnRRUTlSCs z{nE#K=#muD)SZTdy%jL!_62c#`tlWKPHm*9wGwCj zc?r4VFtYHy$onq0?KW?#!v)-KeO!kkWTiPFk5%4U2PTK|O(_rQHk`c7eU0z+Zk_Pi zrzCvs13c!Z`j}3$YthUS9H^qT`f~|W;5`kv(Yn>Y+Gsw9W?ABka1BYQV1%P{HMNGL z@8u)w?qgq;>1eK-Y;Tbgn(YWeU;-*DkVYcwE$s7r1M~OTXZ)1|6h-Vw;g~&4(j+~m zU#K2UTC$3rIqQ~AyZ$nOWj$dTlOLwql@kw(!` z`d`8hO|4jTFNwNIbI}_J36k|e1`%L0;1YNveWKtSr)4{&OL$ZT+n7AwKEFL*)v3`O z274#K4%tTemwf1FBOqg*vs5mZme0atpN37KU9}77FDSLeZQMv%`241D5%^mEpQTy` zCU@S!VSg|5kugYR8BlThTZdDDB;J4^1er{N8f3&Ns)dA(YVz00h1k!fS9B;*hA$gJ z{+aHo;D`X3Wfczsq!%@Zn{xwRT|<$T4_cs~e-BKzdY0czGe-Hl-YLvb+Oof54-0m-T2A>?_JoCg*Vzx6ofeF%um9xrR2$B0B&V@sZ1a z`lIQsi7_LaHW8mk09%J%Cuq5EUk9++=}UmDaado2vkzAp(Oal+-ZOi65OgM4L++Y3 z-VwH3)x>VI8vG`pRVRPd#^#OiotC@y*w0?Mjpo8Elff%RRtKE zWj6Oih2o*DsRu)8qc`v9nuvPUM9qj91wZRbKRS8RKGs|(M^~jG!o|jPM7pBxf$->> zJ*C$|MVU_SF0w#-fZ4CAMdVfY4O{&V$30a*KDkk$QhpFZ@D;e+U&$qaDGfmLARuUD zP+C*HRTfD&9p(GUa!0`6IUrK_&nLd??JUXOwOq3`{}berBaAzb|1yuI_6+dn?=3JxVy`aJ8#25c4sf-W zJ13qw=`|icY{&+h{>WtVs^v}<(TC4J{a?93_TGLzeW}>a-+oc(TvmnhJ~2_39p|V^ zs4%ai62h^-62pF#FRp7Kr1oDE9S^9Ie+;`womatV{>7qo?dZDquOIyZdgEcG{QM$) zo{m|AA4f)W#Cy0dr7P?v-`kp$Q#{~Rus1UZVZDX)&9+{9#)bt~2E{-X795L7}X4R8&D>Y4lz z0DlXI3L*_Ux^unr5{4O;l;NQMR~%6m=$9tM&5!^74yVdVLa&ebcOtByMjcrm1eXH; z@=!_qAmPa)c|C%}&Nafbb>ffWim3|fo>uhVc~G2}ieE7XNGZfR5h>Zg?gXVJMbtaQ zKY2A$dCSN}91O&Z>w5P#JAM3a4oQ34|yC}U+&hH%uMRH6{AM9y9V^`SR zy6eDc)MMk!qa=B!!Qk4h2{$T z*L1Bn3$Amezc@0p2qgplbKfdHN{qwAfSfUM-FPD@bue%jMVcs^kxP;5ITL%{t*1fH zvn2oELXeIZ$xP|4#{?-{u`M?%B4 zel{3UbLt3}aKC+E(QEX*_h$b^UVKJR_;*@QsO+1y|K6y3BJtH}-oy&R)8g9we1Bp1 z7|#cKj%wzp-_ho#P`*vYO6Rv`{WvtOmT{+K4g!_B=gmP* zrs3MwM)Q^B^7`fX%*vqd-rV_AAK6ya_lQkq^=fbvYwpHRxmjfT(;;umlB6tO95hS( zAN4`z>ZUi9^Ao4A&~@mmJ{~6r1$HS%gLNGGKa7^@b3yIXc2UtEWccikqVq-F&sPLu zT%WwIHjI_#4X$3E78T7adU8&v^^3bl2pcaQF5fk{LyRLfR?zf@t9D^m;q^eZc(l$z z;pN|bNnUWYd#g9~a>1K5#XPlCir1xUX<+&<5Jk@f8^{c=vOx&}^XAEk>G<3CA&yw5 z^!{@{80^&B0OB)8g9thiFt4@(ez^;23*6tucbdvr4zM@d*giB-gPGe!Pew4?1sbp> ziN&tS8&W{(@i6DX1DX#R(aD(wyOEsUO2Xmk!o3Jy^EP%9uC$}IBk|^`Bv~T@-uTt*0zcA-MifSrRu8;t`XLQkr3#p4)VS4;E zdxIck3$~9xq!fc}a0Inqg4zC9<36`1>|JrzQAM;>_G=BHhEG%;bZZ5xb?vQ0??G$? zr0c|ed_BZ#mvl1)+$o`gluywcJ4geo9Hat8yw4Ao;Iv_F7e1n`(CY38D!H-kBMWLqoCJ$ z?RSjs+EBV@720aXI+e7LS*A3%}!s z5=+;6Ls<a)5khJ!_<}5wMA>`KLfEsP1a@2{Ut#9- z;clL8szo_uyza&R8W{Y*Xw1N*dw1JS<%wMWhXUO9L&&$cI4B4vH�a7yJtx2MjC^ zOkaIe5)dTkU&@bGkc~Q!e>oE^u7dY^?pt(#MrUf|;oYB1!O_s5E&GW+mk)CUQ>rupnn^#DOQnXmfh*L z((bH#vt~c%wc;&>{|htabRoLGX=Hd6sd*NnZG{_MUjpLnxaYG=YOGp{dGdUIeJh5n z=&15pg?B@;3%@yf#f#i7^29gG4ox7B>!3yuOFjWvw)&f}+CgNlLXEQh$U7Jx`Y(1m z3&+FR5^kV?w)V|_e0~>vP@+dw=#C;u4#8V|@eHTDdP^yDJMkpa>dRG&>~_rJ*o9%q z8(X(}Y~R=FaVe}%hahk{&S}*LOY1D6O|JYul@m<4PaQUUU*a@iN>d!Jr9tnP3c5#F zm^!mD-20tZ``^bhNq#>fyn;F#PIl5;ILPNmq~aFS- zQPpPex_MNBxVhC``!xDuIJtSgI5Z-8{_ET@eJ?y;lThjg;1@{~=( z%2*5-iMo~M-^x;w;)~Gm?mNFm*u=tsZb8%r~pf& zOeyDBX)^sDPSJ-S$z;}>J-JiAN`aNhmNaq$V<4}q->uM6Yeo0xKeJFhP zYLZY%c|VlYMjO|gg5qIF+-IlX%ttwC5plw(#9uQW3$CDi6JR>z&!Q^yR$JhoR3hXe zjw5$fjqUA!(SNW6p-`blvHBW2lio}hz>CeFUg0i+C!qo|%0>oRj{BAhupRqbjnf>a8E z>Kng2mwmadC5(pOc#2Wo`p?VhbHp0#;17eiO8SEzR%5qFF9)R53Zi}EvG91wYxSvc zgxKaj=~FX#HxVJVq}H%!yi(M~apu55JTc#|64-K2$z5SO4A>(~<43+vTnl zP&1oPfNdYw%R@(|_oFxVvcOm`c7}wIy`9+y_9mV$P9SOBSwJA+jjJiuR0xn)AoA2xR3!30B|M>krBy*LFTvO7^HhcT`Z9g)hluzFJ^ha%-qV-|f%E_- zM6o@agESZhGfF=RPsV6{$}l5^Iw^DGAL{9j#T_N>Fj*`i&WPp-#UL*jHz*voNVm=U zv1yVAf>e+lCPYo4h+^6IxliShBP7w6$3f<(hT&h|O~My!}8OhqlxJncx6%o?**$vkN4QyDj`cn`(;*?YpDkVi^MaYq}p8xS5 z1$MJq1~M|BI|E{q-+)5XSv^DE)3L4*%8CWN?a`f!BsGQ&T&uZfohGk5Vb1948ccml z3|FdrCA2)`)T`BPMfsm~*^UZaNUCpusJ7sL^kVvT7Teae<7xWt%=15pS4K42&Ut;$ zAi$5qHnG+??`T`~ILNFxHLi3VXnX2Bp$n_row90PqOhrTo>b*xxKiN#k=3YGiFit% z0_-FDAk^za;yQUTl?k@#4>C4r!$d#6C5nu1k1h?i)V6U#G~HLmZ%z6L!!KYnv)EC~ zL^sIn&u;Vm@34-Kuu}oZ)>P1}_q{fC3n>zVi~C5sRNg0LYf7U9chk$e-6`AVOr!YO z`AQ^FwLOG8^pP}$!f&vV+&&k5f_NL9Ip?X`ijI0I6jFOst~uwNliGrR45p;cg3F;! zYR+e#ZMnN$AuOL9t`xllGe+r|VqJ*K4<{+qF#%_fQ;*}EML4wmXdZ;IYB?#Bj3SrN zF?)d}iSOsGC>0w_n$7&?2-1N7;AE z)Qyws5)ldMO5XJ!(&`YcU3(he86!+OY0%8nY?MVL z>2SGG7XAm=N}mihwWAnkg~2R37%vX;FDy3F{OCqpk94~JN71%Y1}((!AWUQza%i~2g0##kI7y*9@uE27ItphMjSQF}fEbdC z@0;z9pb^kOeBUBj)JK47u^ajzx)yb#F@;Y2*JOsHP$R^-O*DABl>p{TS?-u_l-ev@?QAIcQv)L&G5%v~!Ak2&e5 zP**BfA2~~N;kV{IDpuD!CEYr;m~2ubO*#+|Mx`3*63XimM$`4{xp_CwA0BR1S+nXa z`I`Ok1vP4=116o2Ff@CF0aW=3foA>v4m|pU-<7~=__NjR52!-L4>RZ9A8P~{9$h~2o6^^6_k@Y-lbi`!;y1KaKP(mSxq1|&=Aiq{dW z(XNr*fNX4m>2{SZaINcV)kFa%t*(FqX-~iabBCZ^y^cBA32+EeD8nFeS8}a)6Zty5 zQ#y9%p*tdGSEBTQz=lf2B(z5^ilbiO7<*L0iVV;Z2rUQh@+k;E7{D^UDNuutur?T7 z@-s<>E>dQsOXC{8CYk;tJDL1wV>PQ}nM{g9$$(nFL<=djKU#!XSBniwZXe4EfKnEm ze64f`o3T7M-Ki)J<`LPfBw|FmT3*J5i~*Nkv;ofJs-!DLe7KvfGyk4{QSN(05JK&G zgx|KZ=TU8RSRE@$J&;J_)*V5XoxJ8;=!nFt(M4^PDs`2I$qF7ioR+M1GpO&!!|Bhp z{aZMFx^^EEC#+I8r&T<=Djbp4>pxY)jck1`Rt38IGRk|U zV!phK(nD1FM|by9`g?yNC&_BRC!UZdu2Y#~Ro*gC;A12W5`;Cqr=aHj$}ZWP_n z*kwp6s3qG7J4HOe_r~@Jn~?^Mdbbh@i(=s_?u0jJ6F;ASEkGQ@c9fE&asnty3k)}+ zf|$oBLwbKkha%U(WI{RvKQ#8XrvWS&z0Hkhs--+@Bb5ChyOv@Ov@joSv71al>et~C zNP~7mZD_7AC2z)D_XNVvssc}WuS}BuhrlRFuY+n5!LEEXGs_=DkQcg%SwaZ*^4)k})*6w~u)u0n-PMD1`w$I9VUO3#87W2Y!siL($* z3B&ZA_Psru=JOv>O^8GaV$v1`FzIuWyNKm_p%eV%xt|heUNY#ek#L)NMniJ*rS+-E zlu1b{UN#iwCh6`ISCD)Rn84d|3Ujh!MypBL3^T1DYa=Rq+c*zm!p%S;T7Wq5lNXZz z{>{B_8$%$7@Ws@&Vmu>k=2=o`v%J}vhnJY&^|EZF36h+DsRWvY+I=$g)xQ=uKtVYOXOZ1~H3qF^8T-2N8#!NBY9KO-Ch~U5!zx zg_X~K^_(98cWD8Wfl^JcX=onD;_(=Hq~g7zD?4Sk7;{|t*-VxxFn%aDRAG6@JZD5r zl6-t-B4HbP2PBzL2~^ha1S>+}thBL6Hwqgk=t#I@$r68g-`pvPeC8YnR9bFE_!G}U zwT{eIf}-Lav$uzC6bVt&QDjd( zVm@d*lOY?!*~Anh(3G~!KauT2*=kckJSA`Mg~!Q8i%+v}Ozp}makwFRoz>xR#h#c? zjw%<^fILkprD*xaiTiXlRTJ|>@II_4J78}pp&m(tE*>ft{6fvEwZIrsr7PY14&>F$YDwr8drVI2pxf`Rggh}`?dS9?} zMoo|S#HKr-h|dB^DwTq23U(@MdX~shW%%dR226dGPMDVpi>VROq36ijTsY8<7iE9o z8CT}S6SJF=Z==>1mPm!@6Qrm*QKEq9pAtVx5F~MDo&k*};-HuS^*>21cB+0vyJ!Wc zk#I?h7MB_HXeTaIQS#kuC>-}{S_K}aEF0|VztZR6mQI{mn6DY>c3s%Ax=`8IY9V7`5dX%lijtKG?R{<7;u&xw?)Bm zBcW>;%jucY`@?{?C9>Gn%NDT$oo0L!xyQ`$QzrVIG%eR7;wO zeu3~2pYVPImx`j!Xli?C)NT=Ej248~jTh7Dy%G4gfgL)L@zJwMd&-T;wYA_YNR&Nq zI;A7Spk(lm_LLeh6tJ)gC_?(pO6QUi9nE`j1<_I9eYJcBX6KNe+Qto%am*8V>n1mcMk0!lUkK>DMDH~S zEu+`J%n~dII>EH`6`MH(t%5@^mi)d>FMAwg%p*Yw;gZ6!Js{1LSQEKv=E8T6BBGA7 zq$||9EAP049q}wuX}sX9)MiEn_8ljGrCrxrZIqfKf2cAOmYe-**kGusU=Hj#E?j}i z{!HC;jG8v+lGel~2Pe_7Jb%6rW~u6xIB|$Qi+e3)o=2c@PE!>N$^?k4Q*rAt;3WDD z@xzyVtirRUgixzIQL^T#sNPVEh?EpEj7@D{3LM)lM6KNvqeL?s4lOd&9IwIs2p097GZ#%(Up| zy5Y_$q=$JFJ&fukmm+HKOHFk&)%nV(4o`GA@3>nqyBkO>dGJH%Du@keBRwI~c zw~Y%Q^(NvT7x=C*4~Qorye3_ttpVF>lORs1taRZ2H@v6Sqyrz?7J3}=q-aD*5piy! zwZ%@4Q=hQV9R(qY$Os(SOhfO|STB!c$Q*k>1(M#ldb|$BYe1S`bV?%ZB$qi@_ceWx zF6;FC@aX3Br{NCGp_ptL4w7t}(PELMFrDYq-_v}J$*p)vvIvAq8VsAPdCG;#=I2Xy zBaWv9?ycm%pOTao^mCs@_)Fv_z8r`WuD_UZpN&AH-wVGapDsJeyd^cboD(?3LSfDF zqKzD1G}@k-86LSniZme0WN^b z9iHOaXtY@)i}Ie&)BJ!q(9~vOd&!dcw4Z!y4?w zCZh=d!Tcakq*!!&z;tE7D!DEY0^iGS6arBw{u!0OA5{XU^ zNbcJr6{%i@76a15*G;_0wOj~mp=OrvnbXLG zcj}00p-CD}M}u_h?rP-*rimNUyzBXurgYYJs!WH8LCa#a$Vtf>$zomLN0Y=<~O8t<;$q_u#JCC4M? zlx$ZLYYMF;i|5g?qdVg0Y_w3tLGL>>wPaoJFUP*$QQ2G4eSFKtl{QkLp-h7bNwMZR z-x)Lep43i1`5SW2Y*JF)jDoP#tU^cp8GiQOzTG=M&P+Iw<)T|5)jCqGIu}t*g2Tkb zH3dysCIz8wK~o^ulS#d%_Q#18jgTTMvRdIwAmzZM?yaQ2M*8wgS_3eqfCr*VL6@KK zJ+kDC1pi3BtNZ%M$J&=+-X&?mqaE#}Ds!wc=rJ%Tg-1Q~Nt>s*bBZrgF~w*STfjMz z4~jx5!ab*bo0Mv#2_^~>Poy-9jxPl(rZ}Tu2zfLU@^GQRPHYWm9|GYYs6P?G>=d89 zxjO|#B8l*&0k_#UW_y<{XxGS{tAZ1^%MDquvAHH`;qlo#R&Y{k?JZ}cwDS@YIx`{j z`-ydtP^LlTPJ5U;gxuk5i1Nr=wSY$Gg+C zaoJ}qymZMlF53H&pK$%bn<}XcTlL7hiijUNoQD_dwj^ zsasg(jep_!Q5-HtNs+x&5Ml;4GQZI1ITI*aR7m^V>DgI(Mv_-kvrAbxA+?}j>>muD z{DpSertQafVKDbbO>)x|qtKK{6xm{loflL2x>WUjoz zol%o+7{$fxj5=xO8d7pvk)}}ntkgo2WggD$@X!Nw<)YkWU3d!kv<&OACiH{u2(8 zJCA;OU*Hl_n18sTOlw_80xEJDm(T%zSW%~{I7fMnRcx3O&Do~gaSXD>jHGGgsy~c9 za>Yrnb5lBlRQ6s;VGh$_GFp=jEmcviOKC(p!*HIuVk8JpXF)DF3g&L%fx$#K&;Ig^ z+C8ShE#6yW2UYpqg3Eitn$fQ=sh5D6IbeD>B*_{`sf?P#{LQ^D7)8q4R5A)lpha$x z@}uy{_9{Qgo}^uqgZE35s<|T|FX#|})N1c9P4$^s>co_smsL-T$T#0m;1ZpZmHNEL z6kS+H4WzUYkS@%7NQ5ZB-&8M;zN48$B4UOjSyLI3krau!oK`S8m%*25ptW4CQw5Z7 z%Pv<5JP-m#^KLN#iD)vmM=l|_vAc(^T9$JUoxPDo{I!}S))hr3W%*ZjZnz@IWMUNi z$ej>5=D-J&EV1l+74qZ42et-u(@EHA@+Y#;>?Qu?EbJ*>)sa7#$u^yV*<)1>dbvf9 z7PAHB`T$1b=c?|}1Ud*`OwAIxBxS4}Mc%}pk%HaDk{%`;CmAmaH;cSwMCK#QXG`iG zVTz2g7e*8}NR|w?jGQCAdR*d=TRw=QNNav^3(#Tq(8-hF9&IcyHg`yC%CqW|xx6H+ zx+3P#OX)40aHV(5kKmA|%zH!YFv~J<;%~rG>(P46`Lr(N9T|e-Nn#n0T9H?LmwevD z)Id=xXeF6_n4*?tPeYF{_cN+4AhOwEXx~#mZPabDsWT&7R8{CC=>eo7Enps1w@)Hr zD#lOG8G4FR$W@n7O?x1I^BsyODHF@qD3z@$l>Pn({qvRnaP>Do-&>?qd$iYS_uKuK z&sFc9FJe%6+VlG*=IFrBe!oxt3qQI44h{zU`^huDpdudrbHZxsIC zY#xL$(Eu9k_aI7s3vJ&4tvUJ2O@52+5DU`37Prj(FnL0}*NH%hB!eaarf-tJ>06Mp z@21{u^70b?q%VNnBR^r!6CJieU#ifuZA}9rA0~b9Pd98d8bH#3ONLe)ShpZnv`A*C zSLv9)z&nup_x{Vkx36f;y)nxl1)|7#;JfrGEp&*Eqs9rkR5rK=qWoQGaCvn|*E*dohtyirUiY&M&3K=Ot+UKkKb4(16x zWZm?-`jT|r1rg16rqnNbOcZ;f*TYvNhHKYp!(gl?Tr!>UJwUq6MsI`aci-AcKA+z( zXawuT^))ZAPOsk{S{ElLXQ$^wl1Hq$vy*MlAJI~ae(*3lEH%P1i63#@bfr|sK+S44 zCqV;P3UC_OPLz(2E$^4+`Q=;d=;Hk3^o@0SH9Q$!B@;K_!`kTvS9Lr(Jo=Hg&pxBU z3z2eT0Mc{N>z}vD9m8{fKFem=;enn_setDwn1N{OP?*AY3Lr)Clx+!X$ExD&8n^f8 z0D_xyc^}__?DQ#w#bDwzUjT0k(1$<^)-Xd`H7F(YZiAXP*wWYnK(G3MH@9edx5VEcaJs=GCJ85U;B+0Ex-W&J~>+G0(EJor=c;L0!g4L;)jIW9lxfR%Z#f06)lz;WN=6wKMDJ1vU_ZjFg-FU`8ercYa-VDjT_vxFf!<*BKbL;r@ z3P>+W^UkneHR3L|h!%6gw$Y7#N=>cKEiKk%J9_~J&5fhBa^>uBWJBZEil<$@vPP!p_hDUl~d(TE~5$jsDy6A zCZbs+-E+f>VV}oRUnHg;=2b<+jN0h(kEkQ>k78a@wl^Gc#Fnfc6ErIZ*d_DqC`WOk zSUHMv2g(NGHqndlLz49sET4P=k!^`Q6o;;xcG*prg?awN$Rc2|f)C%{unR8v#|1I5 z)=XW+G~$rzp$KkeP;~+~@}I>Ny>^on&}c~kDoAh*#If)mQ_xQuRUfW`nHmR6SeDqb zkr$-pIT~dYjed~Dh=iMUZAHMHfX?B*y+R_Fljs@Epusdij*=0wb4gI&_UYCU@?*y~ zHseW6Moj@VRrI(m5nV;!1h-M$X=RT&y5f5=nfsbts3CIx$?eXEw;p@Zeq4nh|OsV+nDx0#&K{ZytcD(Vp=-u8PDv*0*4~yHx!5-jO z!FwJB3rC=aYsCof&Wi_5x>uH zQhd)LD8<99WHq+ypX8K00b) zINtQ5TZ}nu=|v3~rLBb)8vrsRfjCF{oR)UtxB7-oHYAw|93|CNs#4^O-==_#NlgnF z$k?OGCObz6`#VepZ-Al&y3L(ye?)tN?X;V@#aEJVdGSZ&PVkccMT(#yhPlgq#kfg( z(xfg*ynSW-=Kj75|Fq#a0~1PrhY$X}^gC=1&0UQuH7uhH*8eVnhTC0WH35{0CJ0-^BI&{+tsX{_6FAC zbJC&+4!ZrqwiDWEXGN(63Wp5j$kGPaX&{8i-_pRp2wJ{1NzLNo7|v?itdzS3jJ6sf%@7ICgCg1=2`)+hA#(C z4t@fA%faSpLc@#r>f=;15JHJc^t|-<)O?HtlJrihnqsH~`bVP^U&hV2%Ic(bJ7c+* zxmOV=%)w&n80Qx^$p2=50qB5{f_pc~iInbkra*ZP(Q9bF7|h!J*&Qvgo)T9siktf| zoJ1ni%k9LC^m&9WFl#77h-|zoLS=J9l3gU?1l=$%qLn-Id$LQ}wh>@TlTRMUzy$=) zV3+#ReV-#zVe>fIc)EDZo$<=(cLyE3HNy*vAc`lUi@KEOM~m?o-7e9+H}pP&h&^$0 zG)9bk7u}kudkt}VrFG;zR z9_lCnt?3z@etS$*_!*)6ncnj>POY$gOB`Dcyd}prx2yS6!1liV7o(z~v!FpLv{)3q zbN}cCB&a}YoRm7NF)m10fQL4wo~*Hm0Pqc|F?lb&KC;oG>V&~OWfL&wtvj+YrUQ*? zywSDRNmp$0Q#S^~U4u8mZeht`^BfesbVo1Lo6!y3(V2 zKMZ=G6!Yc2I~ygjwVh7vNH3(XrRu$0!=e< z0BRE#L(s9qG5*wx_7{Fq3pKS=IJa%0T`Dg!l^#O$ z+VxZDQD5QAY5>n&Isu2vO=*og1@uH|{tzG#{cD=saHMD?8=V`q?C0F61(F+oLQZ`A znjCX_@Qbmb$a>COQi?`p$D!B^YPP2~Xy!&rD*qB^B7R}Ni=QdV6wYm`+o$q`TKgy`U{^GMbnRiO zpd-;Q5bMxI?E4K|#yGb{_b6=TCAWOuwCF9eTw(WjN^bEK>U=Fa^S=1bMnxOS=t`w= zj12C6E_zjis_c~b?#bvvi5QRsiVEo^NAW1l|L_0#zl_{rNN*82dsXvGk~WIso2P4B z8ihi_#>@Xou|H4OB(127DCMq>N*zziubGN&Zmx)~htwr^dqV~hZg044M%hekDqhWy z&f=FzB(7{N{Nxp9*<9Zx@pk zHQ)r+$BY%}P@?srA&uFNXk`GwzK`h6Uxl}utiIxC9t9mOJII^~|^Yht~B_H2%|QJ{R=fragPo8faNL@{H{lzoK}HLTC|!2C7CK}kM8hsp0ic+8Q51ItILk2Qsfq`gH5%=uX1M5 zK|tjK&qZ?;_O30uFMCJKKIr{!=}1-Fq%K!kn!D8GGD~usVy;svNasw%#lm#0VGBlO zb*_lET7pcdWk$B|#rI6LCYHSjHWiIoGghG^XC|wM@j(>tWvqsuL{=#U>#*`fSKTF) z%00~SoY&VR`T2X%k8Cv<6E)|u_{#;b6dzLg&SgykNu`2^eP0PaZ0;-Gjjedq#KPU{ zbY3>ArI!r<2%JYWYKYbyc#6p$7j84)jDR1d_cA31(>WU>=Qmv)!)M`~gs1&Y(_4gyU z1z&!(#qMfPVbLohm1t~IAQt;9wBmW(@|N6^%SA8h#3lLFMkW%beo+z2DbwT2}#BMHzUIVz7de z@BoG3=@7!q;gmarAfF>KBel04N(J002p#21CSv125F1O|;}v@|bK;=}7mvALk+7~> zLsiH%OYL~(gQ{50Zduew+c*{C(=pI4x{qRawrk93K~T(`h(V<)2aR8q&K$ok=eS{< zfo$v>Gc4D#XWYf{JbLx~xf6^qc$*l@+O|jjeol;L_^YLcukebQ|05-5s?VRK$>8_SW1=T zT!M?d&Pfg6oEA}~uoWsya$rGyh3SvWM*`FsGf}4V8Z>-0OKepieO6JOsrvj@nEjQJ zXw!97=z2^scWd~)QEz!GtIlfmy;e7r^)a982S51aH*0)*qoy=K(XLTb6!?GtZ(bD0 z-TdTLZ6Qt~Mb#*u@1}UVOUk>b)r-5vq1QJw?gEAz<_%73*&n8Eh*Axa8{E%cOdas5 z5C5tbn@Su01`G-E5vkpEr*_Mggi^`ulgC7*NZMTU@>MRoI33w z5J-_;jQ0b=`hH(#OKO~$D|e52;%2s1SW}XFFTx5UF`*I*EKXm^2FOrX3m-MU;WBh8 z!wiumNl|rW)T}O*!s}6FxQT1M!N?Yj24CyDXkx(Jc8b`(h|m+FCe^Sa?NB?mTMh{# z`g{=!?H?6EPI45J6urbpEj89o4hmtttfjXexSOjPgm;#Zyu{sH_5R|k62w;JkdgrR zAkl0^`^Z(mrRyv9i+SR=1}iZ7FT1O<|D8;tY1V#v?3+sr&PYTWBgG6Z!RKhn0LvNw>Z<7VZYKU{yrcSzK@@dH-1bQ7P0AOUG(lpb?Xer**X18e$x-7bKfmjf zQW&HZ$?XEG#G*z`T=y~ir8mB>Eg9GmFL5aB2K7~;m2{k10#>6w@cu|@^@x6QR1xxg zG;c*HgHr2!;~`frh0a3g%8zBjqC1vIVuYeMiRamy22oVzJ5=Y|pz4}Hy`@D4spb5? z^G9#oJY@cEza@@u|7n=NXU^Y>Gf3hYqp9o8TQp;O8~;O{M zI)nAFk`H~R3fK#=O82DD#4>K8J0iI(z>9;{Z^H1ur?VOO=T5UI$ELWb|1vs_Vp~eH zzT+>02S&-WeM*=PVIm}+p=}(~V!b z!DU{vaa_BlxEGF#SqwnX@?6*xnvNg4|JV1u_KUW;`@j0_&aUx)z+W%h2iW;`;Y}TO zJBZu;uLbZ=3I#gk-Y5uNfCO!bIuJ~NL$Vq1b$ezzr|+x6e8uqvt0 zNO361KhhNG(JR)9IZHI#?dNQXr07{Y{v5wj;qmw1b3JlHDgz9@H-5|3_`iXJz5hn) z|6Km<&EDzx$;CtGsG$Gd!QdcA|NET-UH@kzKW|x0=)IpPioY_t?SA8&Se;a-p^>(u zNYc1o%&>;YE8~mM{>GU%a{b8NJAolz8E2u zA==vZ2WVG&8Q@v|vuA~$XKOnL<8f8>fv0aFyNlWmH&ts>s$L0EZx0nidM3qHQl`7~ zQg47#wJ5Kxl!q$WOXFeAh$5RvpQfx(z>4w_5oK9+KD}}hH(_pdk3-Uswh8;HIB3O@ zjcxIw&Iqpy{M6?Y Date: Fri, 27 Feb 2026 10:08:44 +0530 Subject: [PATCH 31/54] Fix free models working from UI --- .../src/components/add_model/handle_add_model_submit.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index 8fa5ffd56a2..1d8c980c5ae 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -38,10 +38,11 @@ export const prepareModelAddRequest = async (formValues: Record, ac litellmParamsObj["model"] = mapping.litellm_model; // Handle pricing conversion before processing other fields - if (formValues.input_cost_per_token) { + // Use explicit checks to allow 0 (zero cost models for budget bypass) + if (formValues.input_cost_per_token !== undefined && formValues.input_cost_per_token !== null && formValues.input_cost_per_token !== "") { formValues.input_cost_per_token = Number(formValues.input_cost_per_token) / 1000000; } - if (formValues.output_cost_per_token) { + if (formValues.output_cost_per_token !== undefined && formValues.output_cost_per_token !== null && formValues.output_cost_per_token !== "") { formValues.output_cost_per_token = Number(formValues.output_cost_per_token) / 1000000; } // Keep input_cost_per_second as is, no conversion needed @@ -116,7 +117,7 @@ export const prepareModelAddRequest = async (formValues: Record, ac // Handle the pricing fields else if (key === "input_cost_per_token" || key === "output_cost_per_token" || key === "input_cost_per_second") { - if (value) { + if (value !== undefined && value !== null && value !== "") { litellmParamsObj[key] = Number(value); } continue; From 57c5efc785c2a222f33ac2669c370492f25b5728 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:39:57 -0800 Subject: [PATCH 32/54] refactor: initialize delta vars before try block and avoid redundant find_unique on delete - Initialize teams_to_add/teams_to_remove/keys_to_add/keys_to_remove before the try block in update_access_group for defensive clarity - In delete_access_group, update teams/keys returned by find_many directly (data already fetched) and use _sync_remove only for out-of-sync entities not found by the hasSome query, eliminating N+1 find_unique calls Co-Authored-By: Claude Sonnet 4.6 --- .../access_group_endpoints.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 9a6ff219d4f..a4d0b1104f7 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -394,6 +394,13 @@ async def update_access_group( for field, value in update_fields.items(): update_data[field] = value + # Initialize delta lists before the try block so they remain accessible + # for cache updates after the transaction, even if an error path is added later. + teams_to_add: List[str] = [] + teams_to_remove: List[str] = [] + keys_to_add: List[str] = [] + keys_to_remove: List[str] = [] + try: async with prisma_client.db.tx() as tx: # Read inside the transaction so delta computation is consistent with the write, @@ -495,8 +502,25 @@ async def delete_access_group( ) affected_key_tokens = list(all_affected_key_tokens) - await _sync_remove_access_group_from_teams(tx, affected_team_ids, access_group_id) - await _sync_remove_access_group_from_keys(tx, affected_key_tokens, access_group_id) + # Update teams returned by find_many directly — we already have their data. + for team in teams_with_group: + await tx.litellm_teamtable.update( + where={"team_id": team.team_id}, + data={"access_group_ids": [ag for ag in (team.access_group_ids or []) if ag != access_group_id]}, + ) + # Use _sync_remove only for out-of-sync teams not found by the hasSome query. + out_of_sync_team_ids = set(existing.assigned_team_ids or []) - {t.team_id for t in teams_with_group} + await _sync_remove_access_group_from_teams(tx, list(out_of_sync_team_ids), access_group_id) + + # Update keys returned by find_many directly — we already have their data. + for key in keys_with_group: + await tx.litellm_verificationtoken.update( + where={"token": key.token}, + data={"access_group_ids": [ag for ag in (key.access_group_ids or []) if ag != access_group_id]}, + ) + # Use _sync_remove only for out-of-sync keys not found by the hasSome query. + out_of_sync_key_tokens = set(existing.assigned_key_ids or []) - {k.token for k in keys_with_group} + await _sync_remove_access_group_from_keys(tx, list(out_of_sync_key_tokens), access_group_id) await tx.litellm_accessgrouptable.delete( where={"access_group_id": access_group_id} From 577f7037698450d393b115c715d785f8e7ebbcb4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 10:12:03 +0530 Subject: [PATCH 33/54] Register custom pricing in litellm.model_cost --- litellm/router.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index cbe5b414040..d89a5099b01 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6677,6 +6677,22 @@ class Router: # initialize client self._add_deployment(deployment=deployment) + # Register custom pricing in litellm.model_cost. + # Mirrors _create_deployment() logic to ensure dynamically-added deployments + # (e.g., loaded from DB) also have their custom pricing registered. + # Without this, _is_model_cost_zero() cannot detect explicitly-configured + # zero-cost models, causing budget checks to block free models. + _model_id = deployment.model_info.id + if _model_id is not None: + _model_info_dict: dict = deployment.model_info.model_dump( + exclude_none=True + ) + for field in CustomPricingLiteLLMParams.model_fields.keys(): + field_value = deployment.litellm_params.get(field) + if field_value is not None: + _model_info_dict[field] = field_value + litellm.register_model(model_cost={_model_id: _model_info_dict}) + # add to model names self._add_model_to_list_and_index_map( model=_deployment, model_id=deployment.model_info.id From 06c90ecf62cb8ecd35b8995aef7926fe209d772f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:42:47 -0800 Subject: [PATCH 34/54] Update litellm/proxy/public_endpoints/public_endpoints.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/proxy/public_endpoints/public_endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index d611a454010..ac5d9126145 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -337,8 +337,8 @@ async def get_supported_endpoints() -> SupportedEndpointsResponse: """ global _cached_endpoints if _cached_endpoints is None: - _cached_endpoints = _load_endpoints() - return SupportedEndpointsResponse(endpoints=_cached_endpoints) + _cached_endpoints = SupportedEndpointsResponse(endpoints=_load_endpoints()) + return _cached_endpoints @router.get( From 2144e79bada00324025b0bd2f5083bd6fd644c10 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:48:04 -0800 Subject: [PATCH 35/54] fix: guard against null assigned_*_ids in update_access_group delta computation set(None) raises TypeError when a client sends null for assigned_team_ids or assigned_key_ids. Add `or []` to handle null safely, consistent with create. Add test covering this case. Co-Authored-By: Claude Sonnet 4.6 --- .../access_group_endpoints.py | 4 ++-- .../test_access_group_endpoints.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index a4d0b1104f7..7e75060e87e 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -416,8 +416,8 @@ async def update_access_group( old_team_ids: Set[str] = set(existing.assigned_team_ids or []) old_key_ids: Set[str] = set(existing.assigned_key_ids or []) - new_team_ids: Set[str] = set(update_fields["assigned_team_ids"]) if "assigned_team_ids" in update_fields else old_team_ids - new_key_ids: Set[str] = set(update_fields["assigned_key_ids"]) if "assigned_key_ids" in update_fields else old_key_ids + new_team_ids: Set[str] = set(update_fields["assigned_team_ids"] or []) if "assigned_team_ids" in update_fields else old_team_ids + new_key_ids: Set[str] = set(update_fields["assigned_key_ids"] or []) if "assigned_key_ids" in update_fields else old_key_ids teams_to_add = list(new_team_ids - old_team_ids) teams_to_remove = list(old_team_ids - new_team_ids) diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py index a8842f7448b..fc3c87a112a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -1166,3 +1166,22 @@ def test_delete_access_group_handles_out_of_sync_assigned_keys(client_and_mocks) mock_key_table.find_unique.assert_awaited_once_with(where={"token": "token-out-of-sync"}) mock_key_table.update.assert_not_awaited() + + +def test_update_access_group_null_assigned_ids_treated_as_empty(client_and_mocks): + """Update with explicit null for assigned_*_ids clears the list without TypeError.""" + client, _, mock_table, *_ = client_and_mocks + + existing = _make_access_group_record( + access_group_id="ag-update", + assigned_team_ids=["team-1"], + assigned_key_ids=["key-1"], + ) + mock_table.find_unique = AsyncMock(return_value=existing) + + # Sending null for assigned_team_ids and assigned_key_ids + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_team_ids": None, "assigned_key_ids": None}, + ) + assert resp.status_code == 200 From cde23e9b6eaea89e43f07ef4b7393cd1a424e93b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:53:30 -0800 Subject: [PATCH 36/54] fix: normalize null list fields to [] in update_data before DB write When a client sends null for assigned_team_ids or assigned_key_ids, ensure the DB receives [] instead of null, preventing null from being stored where empty list is expected. Extend test to verify the DB call uses []. Co-Authored-By: Claude Sonnet 4.6 --- .../proxy/management_endpoints/access_group_endpoints.py | 2 ++ .../management_endpoints/test_access_group_endpoints.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 7e75060e87e..53dfbcda836 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -392,6 +392,8 @@ async def update_access_group( update_fields = data.model_dump(exclude_unset=True) update_data: dict = {"updated_by": user_api_key_dict.user_id} for field, value in update_fields.items(): + if field in ("assigned_team_ids", "assigned_key_ids") and value is None: + value = [] update_data[field] = value # Initialize delta lists before the try block so they remain accessible diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py index fc3c87a112a..32fd0750de8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -1169,7 +1169,7 @@ def test_delete_access_group_handles_out_of_sync_assigned_keys(client_and_mocks) def test_update_access_group_null_assigned_ids_treated_as_empty(client_and_mocks): - """Update with explicit null for assigned_*_ids clears the list without TypeError.""" + """Update with explicit null for assigned_*_ids clears the list and writes [] to DB.""" client, _, mock_table, *_ = client_and_mocks existing = _make_access_group_record( @@ -1185,3 +1185,8 @@ def test_update_access_group_null_assigned_ids_treated_as_empty(client_and_mocks json={"assigned_team_ids": None, "assigned_key_ids": None}, ) assert resp.status_code == 200 + + # Verify the DB update was called with [] (not null) for list fields + update_call_kwargs = mock_table.update.call_args.kwargs + assert update_call_kwargs["data"]["assigned_team_ids"] == [] + assert update_call_kwargs["data"]["assigned_key_ids"] == [] From aaf0570f225201f82fe7236355aae70c6d59bcf2 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 21:02:12 -0800 Subject: [PATCH 37/54] fix: normalize null to [] for all Optional[List[str]] fields in update_data Extend the null normalization to access_model_names, access_mcp_server_ids, and access_agent_ids in addition to assigned_team_ids and assigned_key_ids. Writing null for non-optional list fields causes ValidationError on read. Co-Authored-By: Claude Sonnet 4.6 --- litellm/proxy/management_endpoints/access_group_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 53dfbcda836..d58dca5aec0 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -392,7 +392,7 @@ async def update_access_group( update_fields = data.model_dump(exclude_unset=True) update_data: dict = {"updated_by": user_api_key_dict.user_id} for field, value in update_fields.items(): - if field in ("assigned_team_ids", "assigned_key_ids") and value is None: + if field in ("assigned_team_ids", "assigned_key_ids", "access_model_names", "access_mcp_server_ids", "access_agent_ids") and value is None: value = [] update_data[field] = value From 6b9ec4247f51faf42da893e1cc7fe952d2cdface Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 11:31:45 +0530 Subject: [PATCH 38/54] Preserve forwarding server side called tools --- .../prompt_templates/factory.py | 27 ++- tests/llm_translation/test_prompt_factory.py | 175 ++++++++++++++++++ 2 files changed, 194 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index ba415af9a5a..796223ff8e1 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1766,6 +1766,7 @@ def convert_function_to_anthropic_tool_invoke( def convert_to_anthropic_tool_invoke( tool_calls: List[ChatCompletionAssistantToolCall], web_search_results: Optional[List[Any]] = None, + tool_results: Optional[List[Any]] = None, ) -> List[Union[AnthropicMessagesToolUseParam, Dict[str, Any]]]: """ OpenAI tool invokes: @@ -1840,12 +1841,18 @@ def convert_to_anthropic_tool_invoke( } anthropic_tool_invoke.append(_anthropic_server_tool_use) - # Add corresponding web_search_tool_result if available + # Add corresponding tool result if available. + # Check both web_search_results (web_search_tool_result / web_fetch_tool_result) + # and tool_results (bash_code_execution_tool_result, etc.) + _all_tool_results: List[Any] = [] if web_search_results: - for result in web_search_results: - if result.get("tool_use_id") == tool_id: - anthropic_tool_invoke.append(result) - break + _all_tool_results.extend(web_search_results) + if tool_results: + _all_tool_results.extend(tool_results) + for result in _all_tool_results: + if result.get("tool_use_id") == tool_id: + anthropic_tool_invoke.append(result) + break else: # Regular tool_use sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id) @@ -2472,9 +2479,10 @@ def anthropic_messages_pt( # noqa: PLR0915 # Pass through as-is since these are Anthropic-native content types elif m.get("type", "") == "server_tool_use": assistant_content.append(m) # type: ignore - # handle tool_search_tool_result blocks + # handle all *_tool_result blocks (tool_search_tool_result, + # web_search_tool_result, bash_code_execution_tool_result, etc.) # Pass through as-is since these are Anthropic-native content types - elif m.get("type", "") == "tool_search_tool_result": + elif m.get("type", "").endswith("_tool_result"): assistant_content.append(m) # type: ignore elif ( "content" in assistant_content_block @@ -2504,7 +2512,8 @@ def anthropic_messages_pt( # noqa: PLR0915 if ( assistant_tool_calls is not None ): # support assistant tool invoke conversion - # Get web_search_results from provider_specific_fields for server_tool_use reconstruction + # Get web_search_results and tool_results from provider_specific_fields + # for server_tool_use reconstruction. # Fixes: https://github.com/BerriAI/litellm/issues/17737 _provider_specific_fields_raw = assistant_content_block.get( "provider_specific_fields" @@ -2517,9 +2526,11 @@ def anthropic_messages_pt( # noqa: PLR0915 _web_search_results = _provider_specific_fields.get( "web_search_results" ) + _tool_results = _provider_specific_fields.get("tool_results") tool_invoke_results = convert_to_anthropic_tool_invoke( assistant_tool_calls, web_search_results=_web_search_results, + tool_results=_tool_results, ) # Prevent "tool_use ids must be unique" errors by filtering duplicates diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 88bca007740..9f902f2bd86 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -1214,6 +1214,181 @@ def test_anthropic_messages_pt_with_server_tool_use(): assert tool_use["id"] == "toolu_01XYZ789" +def test_convert_to_anthropic_tool_invoke_with_tool_results(): + """ + Test that non-web-search *_tool_result blocks (e.g. bash_code_execution_tool_result) + stored in provider_specific_fields["tool_results"] are paired with their server_tool_use + block when reconstructing assistant history. + + Regression for: server tool result blocks dropped on multi-turn replay + (bash_code_execution_tool_result, text_editor_code_execution_tool_result, etc.) + """ + tool_calls = [ + { + "id": "srvtoolu_01BASH", + "type": "function", + "function": { + "name": "bash_code_execution", + "arguments": '{"command": "python3 -c \\"print(2)\\""}', + }, + } + ] + + tool_results = [ + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01BASH", + "content": { + "type": "bash_code_execution_result", + "stdout": "2\n", + "stderr": "", + "return_code": 0, + "content": [], + }, + } + ] + + result = convert_to_anthropic_tool_invoke(tool_calls, tool_results=tool_results) + + assert len(result) == 2 + # First: server_tool_use + assert result[0]["type"] == "server_tool_use" + assert result[0]["id"] == "srvtoolu_01BASH" + assert result[0]["name"] == "bash_code_execution" + # Second: bash_code_execution_tool_result paired correctly + assert result[1]["type"] == "bash_code_execution_tool_result" + assert result[1]["tool_use_id"] == "srvtoolu_01BASH" + + +def test_anthropic_messages_pt_raw_bash_tool_result_passthrough(): + """ + Test that raw assistant content lists containing bash_code_execution_tool_result + blocks are passed through intact to Anthropic. + + Regression: the raw-block passthrough only handled tool_search_tool_result; + bash_code_execution_tool_result and other *_tool_result types were silently dropped. + """ + messages = [ + {"role": "user", "content": "What is 1+1?"}, + { + "role": "assistant", + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_01BASH", + "name": "bash_code_execution", + "input": {"command": "python3 -c \"print(1+1)\""}, + }, + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01BASH", + "content": { + "type": "bash_code_execution_result", + "stdout": "2\n", + "stderr": "", + "return_code": 0, + "content": [], + }, + }, + {"type": "text", "text": "The answer is 2."}, + ], + }, + {"role": "user", "content": "Thanks!"}, + ] + + result = anthropic_messages_pt( + messages, model="claude-sonnet-4-5", llm_provider="anthropic" + ) + + assistant_msg = next(m for m in result if m["role"] == "assistant") + content = assistant_msg["content"] + types = [c.get("type") for c in content] + + assert "server_tool_use" in types, "server_tool_use block must be preserved" + assert ( + "bash_code_execution_tool_result" in types + ), "bash_code_execution_tool_result block must not be dropped" + assert "text" in types + + # Result must immediately follow its server_tool_use + srv_idx = types.index("server_tool_use") + result_idx = types.index("bash_code_execution_tool_result") + assert result_idx == srv_idx + 1 + + bash_result = next( + c for c in content if c.get("type") == "bash_code_execution_tool_result" + ) + assert bash_result["tool_use_id"] == "srvtoolu_01BASH" + + +def test_anthropic_messages_pt_with_bash_tool_result_in_provider_specific_fields(): + """ + Test that anthropic_messages_pt correctly reconstructs bash_code_execution_tool_result + from provider_specific_fields["tool_results"] when replaying LiteLLM response objects. + + Regression: only web_search_results were read from provider_specific_fields; + tool_results (bash_code_execution_tool_result, etc.) were silently lost. + """ + messages = [ + {"role": "user", "content": "What is 1+1?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "srvtoolu_01BASH", + "type": "function", + "function": { + "name": "bash_code_execution", + "arguments": '{"command": "python3 -c \\"print(1+1)\\""}', + }, + } + ], + "provider_specific_fields": { + "tool_results": [ + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01BASH", + "content": { + "type": "bash_code_execution_result", + "stdout": "2\n", + "stderr": "", + "return_code": 0, + "content": [], + }, + } + ] + }, + }, + {"role": "user", "content": "Thanks!"}, + ] + + result = anthropic_messages_pt( + messages, model="claude-sonnet-4-5", llm_provider="anthropic" + ) + + assistant_msg = next(m for m in result if m["role"] == "assistant") + content = assistant_msg["content"] + types = [c.get("type") for c in content] + + assert "server_tool_use" in types, "server_tool_use block must be reconstructed" + assert ( + "bash_code_execution_tool_result" in types + ), "bash_code_execution_tool_result must be paired from provider_specific_fields['tool_results']" + + # Result must immediately follow its server_tool_use + srv_idx = types.index("server_tool_use") + result_idx = types.index("bash_code_execution_tool_result") + assert result_idx == srv_idx + 1 + + srv = next(c for c in content if c.get("type") == "server_tool_use") + assert srv["id"] == "srvtoolu_01BASH" + bash_result = next( + c for c in content if c.get("type") == "bash_code_execution_tool_result" + ) + assert bash_result["tool_use_id"] == "srvtoolu_01BASH" + + # ============ parse_tool_call_arguments Tests ============ # Tests for the shared utility that parses tool call JSON arguments From 596437b3b901db2f2b01e8a134d308fbaf46b4c4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 12:04:12 +0530 Subject: [PATCH 39/54] Add Regression tests for image_url blocks in assistant message content. --- .../types/llms/test_types_llms_openai.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/test_litellm/types/llms/test_types_llms_openai.py b/tests/test_litellm/types/llms/test_types_llms_openai.py index 87cc9586665..054fe505764 100644 --- a/tests/test_litellm/types/llms/test_types_llms_openai.py +++ b/tests/test_litellm/types/llms/test_types_llms_openai.py @@ -169,3 +169,97 @@ class TestResponsesAPIResponseOutputText: ) assert response.output_text == "" + + +class TestAssistantMessageImageUrlContent: + """ + Regression tests for image_url blocks in assistant message content. + + Bug: ChatCompletionAssistantMessage.content did not include + ChatCompletionImageObject in its union, so Pydantic v2 silently dropped + image_url blocks (content → []) when serialising via AllMessageValues. + This affects users who store conversation history as JSON (e.g. in a DB) + and read it back typed as list[AllMessageValues]. + """ + + ASSISTANT_MESSAGE_WITH_IMAGE = { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here is the image you requested:"}, + { + "type": "image_url", + "image_url": { + "url": ( + "data:image/png;base64," + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAA" + "DUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + ) + }, + }, + ], + } + + def test_assistant_message_image_url_preserved_single(self): + """ + TypeAdapter(ChatCompletionAssistantMessage): image_url block must survive + validate_python → dump_python without being dropped or raising an error. + """ + from typing import List + + from pydantic import TypeAdapter + + from litellm.types.llms.openai import ChatCompletionAssistantMessage + + adapter = TypeAdapter(ChatCompletionAssistantMessage) + validated = adapter.validate_python(self.ASSISTANT_MESSAGE_WITH_IMAGE) + dumped = adapter.dump_python(validated) + + raw_content = dumped.get("content") + # Pydantic may return a lazy SerializationIterator for Iterable fields; + # convert to list to consume it — this must not raise ValidationError. + content_blocks = list(raw_content) if raw_content is not None else [] + + assert len(content_blocks) == 2, ( + f"Expected 2 content blocks (text + image_url), got {len(content_blocks)}: {content_blocks}" + ) + types = [b.get("type") for b in content_blocks if isinstance(b, dict)] + assert "image_url" in types, f"image_url block was silently dropped; blocks: {content_blocks}" + + def test_assistant_message_image_url_preserved_in_all_message_values(self): + """ + TypeAdapter(List[AllMessageValues]) DB round-trip: image_url blocks in an + assistant message must not be silently dropped during dump_python(mode='json'). + + This is the primary failing path: conversation history stored as JSON in a + database and read back typed as list[AllMessageValues]. + """ + from typing import List + + from pydantic import TypeAdapter + + from litellm.types.llms.openai import AllMessageValues + + conversation = [ + { + "role": "user", + "content": "Generate an image of a banana wearing a LiteLLM costume", + }, + self.ASSISTANT_MESSAGE_WITH_IMAGE, + ] + + adapter = TypeAdapter(List[AllMessageValues]) + validated = adapter.validate_python(conversation) + dumped = adapter.dump_python(validated, mode="json") + + assistant = next((m for m in dumped if m.get("role") == "assistant"), None) + assert assistant is not None, "Assistant message missing after serialisation" + + content = assistant.get("content", []) + assert isinstance(content, list), f"content should be a list, got {type(content)}" + assert len(content) == 2, ( + f"Expected 2 content blocks (text + image_url), got {len(content)}: {content}" + ) + types = [b.get("type") for b in content if isinstance(b, dict)] + assert "image_url" in types, ( + f"image_url block was silently dropped during AllMessageValues serialisation; blocks: {content}" + ) From 8565c70e539dd307918a430926330c0d285cb19b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 12:45:08 +0530 Subject: [PATCH 40/54] Revert "Fix mapping of parallel_tool_calls for bedrock converse" --- .../bedrock/chat/converse_transformation.py | 19 ------ .../chat/test_converse_transformation.py | 66 +------------------ 2 files changed, 3 insertions(+), 82 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d4fd0606302..62081114061 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -511,7 +511,6 @@ class AmazonConverseConfig(BaseConfig): "response_format", "requestMetadata", "service_tier", - "parallel_tool_calls", ] if ( @@ -914,13 +913,6 @@ class AmazonConverseConfig(BaseConfig): ) if _tool_choice_value is not None: optional_params["tool_choice"] = _tool_choice_value - if param == "parallel_tool_calls": - disable_parallel = not value - optional_params["_parallel_tool_use_config"] = { - "tool_choice": { - "disable_parallel_tool_use": disable_parallel - } - } if param == "thinking": optional_params["thinking"] = value elif param == "reasoning_effort" and isinstance(value, str): @@ -1215,17 +1207,6 @@ class AmazonConverseConfig(BaseConfig): k: v for k, v in inference_params.items() if k in total_supported_params } - # Handle parallel_tool_calls configuration - parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None) - if parallel_tool_use_config is not None: - # Merge the tool_choice config from parallel_tool_calls into additional_request_params - for key, value in parallel_tool_use_config.items(): - if key in additional_request_params and isinstance(additional_request_params[key], dict) and isinstance(value, dict): - # Merge dictionaries - additional_request_params[key].update(value) - else: - additional_request_params[key] = value - # Only set the topK value in for models that support it additional_request_params.update( self._handle_top_k_value(model, inference_params) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 26395597166..c773db21074 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -2616,11 +2616,11 @@ def test_empty_assistant_message_handling(): empty or whitespace-only content with a placeholder to prevent AWS Bedrock Converse API 400 Bad Request errors. """ - # Import the litellm module that factory.py uses to ensure we patch the correct reference - import litellm.litellm_core_utils.prompt_templates.factory as factory_module from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, ) + # Import the litellm module that factory.py uses to ensure we patch the correct reference + import litellm.litellm_core_utils.prompt_templates.factory as factory_module # Test case 1: Empty string content - test with modify_params=True to prevent merging messages = [ @@ -3135,12 +3135,7 @@ def test_native_structured_output_no_fake_stream(): def test_transform_request_with_output_config(): """Test that outputConfig flows through _transform_request_helper into the final request.""" - from litellm.types.llms.bedrock import ( - JsonSchemaDefinition, - OutputConfigBlock, - OutputFormat, - OutputFormatStructure, - ) + from litellm.types.llms.bedrock import OutputConfigBlock, OutputFormat, OutputFormatStructure, JsonSchemaDefinition config = AmazonConverseConfig() @@ -3382,61 +3377,6 @@ def test_output_config_applies_additional_properties(): -def test_parallel_tool_calls_in_request_transformation(): - """Test that parallel_tool_calls is correctly placed in additionalModelRequestFields after full transformation""" - config = AmazonConverseConfig() - - messages = [ - {"role": "user", "content": "What's the weather in SF and NYC?"} - ] - - non_default_params = { - "parallel_tool_calls": False, - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The location to get weather for" - } - }, - "required": ["location"] - } - } - } - ], - "max_tokens": 100, - } - - optional_params = config.map_openai_params( - non_default_params=non_default_params, - optional_params={}, - model="anthropic.claude-sonnet-4-5-v2:0", - drop_params=False, - ) - - # Transform the request - request_data = config.transform_request( - model="anthropic.claude-sonnet-4-5-v2:0", - messages=messages, - optional_params=optional_params, - litellm_params={}, - headers={}, - ) - - # Verify the structure - assert "additionalModelRequestFields" in request_data - assert "tool_choice" in request_data["additionalModelRequestFields"] - assert "disable_parallel_tool_use" in request_data["additionalModelRequestFields"]["tool_choice"] - assert request_data["additionalModelRequestFields"]["tool_choice"]["disable_parallel_tool_use"] is True - - class TestBedrockMinThinkingBudgetTokens: """Test that thinking.budget_tokens is clamped to the Bedrock minimum (1024).""" From d0445e1e33590963018f0be4b644c4f4905fae0a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 13:00:57 +0530 Subject: [PATCH 41/54] Fix converse handling for parallel_tool_calls --- .../bedrock/chat/converse_transformation.py | 6 +- .../chat/test_converse_transformation.py | 95 +++++++++++-------- 2 files changed, 60 insertions(+), 41 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d4fd0606302..a0f2f65fb7f 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1217,15 +1217,15 @@ class AmazonConverseConfig(BaseConfig): # Handle parallel_tool_calls configuration parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None) - if parallel_tool_use_config is not None: - # Merge the tool_choice config from parallel_tool_calls into additional_request_params + if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model): for key, value in parallel_tool_use_config.items(): if key in additional_request_params and isinstance(additional_request_params[key], dict) and isinstance(value, dict): - # Merge dictionaries additional_request_params[key].update(value) else: additional_request_params[key] = value + additional_request_params.pop("parallel_tool_calls", None) + # Only set the topK value in for models that support it additional_request_params.update( self._handle_top_k_value(model, inference_params) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 26395597166..2b996977d8d 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -3382,59 +3382,78 @@ def test_output_config_applies_additional_properties(): -def test_parallel_tool_calls_in_request_transformation(): - """Test that parallel_tool_calls is correctly placed in additionalModelRequestFields after full transformation""" - config = AmazonConverseConfig() - - messages = [ - {"role": "user", "content": "What's the weather in SF and NYC?"} - ] - - non_default_params = { - "parallel_tool_calls": False, - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The location to get weather for" - } - }, - "required": ["location"] +_TOOL_PARAM = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The location to get weather for", } - } - } - ], - "max_tokens": 100, + }, + "required": ["location"], + }, + }, } - +] + + +def test_parallel_tool_calls_newer_model_adds_disable_flag(): + """Newer Claude models (4.5+) should get disable_parallel_tool_use in additionalModelRequestFields.""" + config = AmazonConverseConfig() + model = "anthropic.claude-sonnet-4-5-20250929-v1:0" + messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}] + optional_params = config.map_openai_params( - non_default_params=non_default_params, + non_default_params={"parallel_tool_calls": False, "tools": _TOOL_PARAM}, optional_params={}, - model="anthropic.claude-sonnet-4-5-v2:0", + model=model, drop_params=False, ) - - # Transform the request + request_data = config.transform_request( - model="anthropic.claude-sonnet-4-5-v2:0", + model=model, messages=messages, optional_params=optional_params, litellm_params={}, headers={}, ) - - # Verify the structure + assert "additionalModelRequestFields" in request_data assert "tool_choice" in request_data["additionalModelRequestFields"] - assert "disable_parallel_tool_use" in request_data["additionalModelRequestFields"]["tool_choice"] assert request_data["additionalModelRequestFields"]["tool_choice"]["disable_parallel_tool_use"] is True + assert "parallel_tool_calls" not in request_data["additionalModelRequestFields"] + + +def test_parallel_tool_calls_older_model_drops_disable_flag(): + """Older Claude models (pre-4.5) must NOT receive disable_parallel_tool_use — Bedrock rejects it.""" + config = AmazonConverseConfig() + model = "anthropic.claude-3-5-sonnet-20241022-v2:0" + messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}] + + optional_params = config.map_openai_params( + non_default_params={"parallel_tool_calls": False, "tools": _TOOL_PARAM}, + optional_params={}, + model=model, + drop_params=False, + ) + + request_data = config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + additional = request_data.get("additionalModelRequestFields", {}) + assert "tool_choice" not in additional + assert "parallel_tool_calls" not in additional class TestBedrockMinThinkingBudgetTokens: From 99c62ca40ea82ab31d61a251410d1a0367ae7aff Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 13:26:31 +0530 Subject: [PATCH 42/54] Add opt out varible for v1/messages to responses --- litellm/__init__.py | 3 +++ .../experimental_pass_through/messages/handler.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 6e42f2c1ea5..50fa0e76755 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -197,6 +197,9 @@ telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) +use_chat_completions_url_for_anthropic_messages: bool = bool( + os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) +) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API retry = True ### AUTH ### api_key: Optional[str] = None diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 6fe0fcd4fdf..5b215c1fe54 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -30,11 +30,17 @@ from .utils import AnthropicMessagesRequestUtils, mock_response # Providers that are routed directly to the OpenAI Responses API instead of # going through chat/completions. -_RESPONSES_API_PROVIDERS = frozenset({"openai", "azure", "azure_text"}) +_RESPONSES_API_PROVIDERS = frozenset({"openai"}) def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool: - """Return True when the provider should use the Responses API path.""" + """Return True when the provider should use the Responses API path. + + Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to + opt out and route OpenAI/Azure requests through chat/completions instead. + """ + if litellm.use_chat_completions_url_for_anthropic_messages: + return False return custom_llm_provider in _RESPONSES_API_PROVIDERS ####### ENVIRONMENT VARIABLES ################### From 2fa9b81e2fb5aae7ad8dc74663b6b5b3de6696c0 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 13:28:48 +0530 Subject: [PATCH 43/54] Add docs for opt out variable --- docs/my-website/docs/proxy/config_settings.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 8dbebad884e..4f862cf8471 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -196,6 +196,7 @@ router_settings: | disable_end_user_cost_tracking_prometheus_only | boolean | If true, turns off end user cost tracking on prometheus metrics only. | | key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) | | disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. | +| use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. | | disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). | | enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. | | disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. | From a9d0e2cf91663a5bcc7ddfffaf8b4683b3ee5c31 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Fri, 27 Feb 2026 13:33:34 +0530 Subject: [PATCH 44/54] fix: req changes --- litellm/proxy/_types.py | 10 ++------- .../management_endpoints/project_endpoints.py | 22 ++++++++++++++++++- .../test_project_tags_pydantic.py | 16 +++++--------- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8713f54a3ef..28311ab1b3b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2599,10 +2599,7 @@ class NewProjectRequest(LiteLLM_BudgetTable): raise ValueError( f"tags must be a list of strings, got {type(values['tags']).__name__}" ) - for field in ( - LiteLLM_ManagementEndpoint_MetadataFields - + LiteLLM_ManagementEndpoint_MetadataFields_Premium - ): + for field in LiteLLM_ManagementEndpoint_MetadataFields: if values.get(field) is not None: if values.get("metadata") is None: values.update({"metadata": {}}) @@ -2635,10 +2632,7 @@ class UpdateProjectRequest(LiteLLM_BudgetTable): raise ValueError( f"tags must be a list of strings, got {type(values['tags']).__name__}" ) - for field in ( - LiteLLM_ManagementEndpoint_MetadataFields - + LiteLLM_ManagementEndpoint_MetadataFields_Premium - ): + for field in LiteLLM_ManagementEndpoint_MetadataFields: if values.get(field) is not None: if values.get("metadata") is None: values.update({"metadata": {}}) diff --git a/litellm/proxy/management_endpoints/project_endpoints.py b/litellm/proxy/management_endpoints/project_endpoints.py index c825802f314..8f48f9def78 100644 --- a/litellm/proxy/management_endpoints/project_endpoints.py +++ b/litellm/proxy/management_endpoints/project_endpoints.py @@ -358,6 +358,16 @@ async def new_project( }, ) + # ADD METADATA FIELDS + for field in LiteLLM_ManagementEndpoint_MetadataFields_Premium: + if getattr(data, field, None) is not None: + _set_object_metadata_field( + object_data=data, + field_name=field, + value=getattr(data, field), + ) + delattr(data, field) + if prisma_client is None: raise HTTPException( status_code=500, @@ -473,7 +483,7 @@ async def new_project( response_model=LiteLLM_ProjectTable, ) @management_endpoint_wrapper -async def update_project( +async def update_project( # noqa: PLR0915 data: UpdateProjectRequest, http_request: Request, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -543,6 +553,16 @@ async def update_project( }, ) + # ADD METADATA FIELDS + for field in LiteLLM_ManagementEndpoint_MetadataFields_Premium: + if getattr(data, field, None) is not None: + _set_object_metadata_field( + object_data=data, + field_name=field, + value=getattr(data, field), + ) + delattr(data, field) + if prisma_client is None: raise HTTPException( status_code=500, diff --git a/tests/test_litellm/test_project_tags_pydantic.py b/tests/test_litellm/test_project_tags_pydantic.py index ed3b29fe7be..b3f58df2325 100644 --- a/tests/test_litellm/test_project_tags_pydantic.py +++ b/tests/test_litellm/test_project_tags_pydantic.py @@ -3,26 +3,20 @@ from litellm.proxy._types import NewProjectRequest, UpdateProjectRequest def test_new_project_request_tags(): - # Test tags are correctly moved to metadata["tags"] + # Test tags correctly stay top level initially req = NewProjectRequest( project_id="test_proj", team_id="team_1", tags=["tag1", "tag2"] ) - # After validation, tags should be inside metadata - assert req.metadata is not None - assert "tags" in req.metadata - assert req.metadata["tags"] == ["tag1", "tag2"] - assert req.tags is None # Or removed dependending on pydantic version + # tags should be top level initially + assert req.tags == ["tag1", "tag2"] def test_update_project_request_tags(): - # Test tags are correctly moved to metadata["tags"] + # Test tags correctly stay top level initially req = UpdateProjectRequest(project_id="test_proj", tags=["new_tag"]) - assert req.metadata is not None - assert "tags" in req.metadata - assert req.metadata["tags"] == ["new_tag"] - assert req.tags is None + assert req.tags == ["new_tag"] def test_new_project_request_invalid_tags_type(): From 72d7f64345a40be62b4f0aaa6e1f49b2cbd0f48c Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Fri, 27 Feb 2026 13:52:50 +0530 Subject: [PATCH 45/54] fix: relevant comment req changes --- litellm/proxy/litellm_pre_call_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index eac1c4a33c6..3168bcd812f 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1023,7 +1023,7 @@ async def add_litellm_data_to_request( # noqa: PLR0915 "spend_logs_metadata" ] - ## PROJECT-LEVEL SPEND LOGS/TAGS + ## PROJECT-LEVEL TAGS project_metadata = user_api_key_dict.project_metadata or {} if "tags" in project_metadata and project_metadata["tags"] is not None: data[_metadata_variable_name]["tags"] = LiteLLMProxyRequestSetup._merge_tags( From 3bec6f5a9a349a2b55273511e93c83f4247dd947 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 21:39:32 +0530 Subject: [PATCH 46/54] Fix: poetry lock --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 34227a69ccb..0314a360542 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3222,15 +3222,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.48" +version = "0.4.49" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.48-py3-none-any.whl", hash = "sha256:097001fccec5dbf4cffd902114898a9cfeba62673202447d55d2d0286cf93126"}, - {file = "litellm_proxy_extras-0.4.48.tar.gz", hash = "sha256:5d5d8acf31b92d0cd6738555fb4a2411819755155438de9fb23c724c356400a2"}, + {file = "litellm_proxy_extras-0.4.49-py3-none-any.whl", hash = "sha256:aeb0e08b4705c19fdc5b75a43c608a82fc36032f6d83be509dbf37baea62f2cd"}, + {file = "litellm_proxy_extras-0.4.49.tar.gz", hash = "sha256:d9bdae54d1e3398f2e2025c9d8b98a19e226874337d540d5415922d7dbbc97bb"}, ] [[package]] @@ -7989,4 +7989,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "b9b1e47b3b84748c0053be6a544c2399bf2601746a4f88dcb1be7c5e4eeab359" +content-hash = "bbc7d43f5484af4c8877fe66e34f8283069528379af49d573036ba144cc2eb7a" From d13508c1c56074a894ffd365b615037353c92d2b Mon Sep 17 00:00:00 2001 From: Noah Nistler <60981020+noahnistler@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:50:02 -0600 Subject: [PATCH 47/54] Enable local file support for OCR (#22133) * [Docs] Enable local file support Implemented internal handling for converting file-type documents to the required format for OCR processing, ensuring seamless integration with various providers. * Refactor OCR file handling and improve security checks Removed deprecated MIME type mapping and file conversion functions, replacing them with updated implementations. Enhanced security by rejecting 'file' document types in JSON requests, ensuring file uploads are handled via multipart/form-data. Updated tests to reflect these changes and ensure proper functionality. * Enhance MIME type validation in OCR processing Added a regular expression check to validate MIME types in the convert_file_document_to_url_document function, raising a ValueError for invalid types. Updated tests to ensure proper error handling for unsupported MIME types. * Enhance type safety in OCR file handling Added type casting for the uploaded file in the _parse_multipart_form function to ensure proper handling of UploadFile instances. This change improves type safety and reduces potential runtime errors during file processing. * Refactor MIME type handling in document uploads Updated the MIME type extraction logic to strip parameters from the Content-Type header, ensuring only the base type is used. Added tests to verify that MIME parameters are correctly handled and stripped in various scenarios. * Update OCR documentation for MIME type recommendations and remove unnecessary tips Clarified the recommended usage of MIME types for raw bytes in document uploads. Simplified the documentation by removing the tip about multipart file uploads from tools like Postman, ensuring a more concise and focused guide. * Enhance multipart form handling in OCR endpoints Updated the _parse_multipart_form function to ignore both 'file' and 'document' fields during form parsing, ensuring that the document built from the uploaded file is not overridden. Added a new test to verify that injected document fields do not affect the constructed document, improving security and robustness of the file upload process. --- docs/my-website/docs/ocr.md | 98 +++- litellm/llms/base_llm/ocr/transformation.py | 10 +- litellm/ocr/main.py | 226 +++++++-- litellm/proxy/ocr_endpoints/endpoints.py | 206 +++++++- tests/test_litellm/ocr/__init__.py | 0 tests/test_litellm/ocr/test_ocr_file_input.py | 464 ++++++++++++++++++ 6 files changed, 939 insertions(+), 65 deletions(-) create mode 100644 tests/test_litellm/ocr/__init__.py create mode 100644 tests/test_litellm/ocr/test_ocr_file_input.py diff --git a/docs/my-website/docs/ocr.md b/docs/my-website/docs/ocr.md index 93cb74ee69f..cea6fce1254 100644 --- a/docs/my-website/docs/ocr.md +++ b/docs/my-website/docs/ocr.md @@ -61,6 +61,52 @@ async def test_async_ocr(): asyncio.run(test_async_ocr()) ``` +### Using Local Files + +LiteLLM can read local files directly — no manual base64 encoding needed: + +```python +from litellm import ocr + +# OCR with a local PDF file path +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "file", + "file": "/path/to/document.pdf" + } +) + +# OCR with a file object +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "file", + "file": open("document.pdf", "rb") + } +) + +# OCR with raw bytes +with open("document.pdf", "rb") as f: + pdf_bytes = f.read() + +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "file", + "file": pdf_bytes, + "mime_type": "application/pdf" # recommended for raw bytes (auto-detected from extension for file paths) + } +) +``` + +The `file` field accepts: +- **File path** (`str` or `pathlib.Path`) — LiteLLM reads the file and detects the MIME type from the extension +- **File object** (binary file-like object) — e.g. `open("doc.pdf", "rb")` +- **Raw bytes** (`bytes`) — use `mime_type` to specify the content type + +LiteLLM automatically converts file inputs to base64 data URIs internally, so all providers work seamlessly. + ### Using Base64 Encoded Documents ```python @@ -121,7 +167,7 @@ litellm --config /path/to/config.yaml # RUNNING on http://0.0.0.0:4000 ``` -Test request +**Test request — JSON body** ```bash curl http://0.0.0.0:4000/v1/ocr \ @@ -136,6 +182,27 @@ curl http://0.0.0.0:4000/v1/ocr \ }' ``` +**Test request — multipart file upload** + +Upload a file directly using multipart form data. No need to base64-encode the file yourself. + +```bash +curl http://0.0.0.0:4000/v1/ocr \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@/path/to/document.pdf" +``` + +You can also pass optional parameters as additional form fields: + +```bash +curl http://0.0.0.0:4000/v1/ocr \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@screenshot.png" \ + -F 'pages=[0,1,2]' \ + -F "include_image_base64=true" +``` ## **Request/Response Format** @@ -168,10 +235,12 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie | Parameter | Type | Required | Description | |-----------|------|----------|-------------| | `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) | -| `document` | object | Yes | Document to process. Must contain `type` and URL field | -| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images | -| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) | -| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) | +| `document` | object | Yes | Document to process. Must contain `type` and the corresponding field | +| `document.type` | string | Yes | `"document_url"` for PDFs/docs, `"image_url"` for images, or `"file"` for local files | +| `document.document_url` | string | Conditional | URL or data URI to the document (required if `type` is `"document_url"`) | +| `document.image_url` | string | Conditional | URL or data URI to the image (required if `type` is `"image_url"`) | +| `document.file` | string/bytes/file | Conditional | File path, bytes, or file-like object (required if `type` is `"file"`) | +| `document.mime_type` | string | No | Explicit MIME type for file inputs (auto-detected from extension if not provided) | | `pages` | array | No | List of specific page indices to process (0-indexed) | | `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings | | `image_limit` | integer | No | Maximum number of images to return | @@ -179,7 +248,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie #### Document Format Examples -**For PDFs and documents:** +**For PDFs and documents (URL):** ```json { "type": "document_url", @@ -187,7 +256,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie } ``` -**For images:** +**For images (URL):** ```json { "type": "image_url", @@ -203,6 +272,21 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie } ``` +**For local files (SDK):** +```python +{"type": "file", "file": "/path/to/document.pdf"} +{"type": "file", "file": open("image.png", "rb")} +{"type": "file", "file": pdf_bytes, "mime_type": "application/pdf"} +``` + +**For file uploads (Proxy — multipart form):** +```bash +curl http://0.0.0.0:4000/v1/ocr \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@document.pdf" +``` + ### Response Format The response follows Mistral's OCR format with the following structure: diff --git a/litellm/llms/base_llm/ocr/transformation.py b/litellm/llms/base_llm/ocr/transformation.py index fb13332c464..29929a2bf62 100644 --- a/litellm/llms/base_llm/ocr/transformation.py +++ b/litellm/llms/base_llm/ocr/transformation.py @@ -15,7 +15,9 @@ else: LiteLLMLoggingObj = Any -# DocumentType for OCR - Mistral format document dict +# DocumentType for OCR - providers always receive a dict with +# type="document_url" or type="image_url" (str values only). +# File-type inputs are preprocessed to this format in litellm/ocr/main.py. DocumentType = Dict[str, str] @@ -141,9 +143,13 @@ class BaseOCRConfig: Transform OCR request to provider-specific format. Override in provider-specific implementations. + Note: By the time this method is called, any file-type documents have already + been converted to document_url/image_url format with base64 data URIs by + the preprocessing in litellm/ocr/main.py. + Args: model: Model name - document: Document to process (Mistral format dict, or file path, bytes, etc.) + document: Document to process - always a dict with type="document_url" or type="image_url" optional_params: Optional parameters for the request headers: Request headers diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 5acab8cbf2c..47cff8a2c0c 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -2,8 +2,14 @@ Main OCR function for LiteLLM. """ import asyncio +import base64 import contextvars +import mimetypes +import os +import re from functools import partial +from io import IOBase +from pathlib import Path from typing import Any, Coroutine, Dict, Optional, Union import httpx @@ -25,7 +31,7 @@ base_llm_http_handler = BaseLLMHTTPHandler() @client async def aocr( model: str, - document: Dict[str, str], + document: Dict[str, Any], api_key: Optional[str] = None, api_base: Optional[str] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, @@ -35,26 +41,27 @@ async def aocr( ) -> OCRResponse: """ Async OCR function. - + Args: model: Model name (e.g., "mistral/mistral-ocr-latest") document: Document to process in Mistral format: - {"type": "document_url", "document_url": "https://..."} for PDFs/docs or - {"type": "image_url", "image_url": "https://..."} for images + {"type": "document_url", "document_url": "https://..."} for PDFs/docs, + {"type": "image_url", "image_url": "https://..."} for images, or + {"type": "file", "file": } for local files api_key: Optional API key api_base: Optional API base URL timeout: Optional timeout custom_llm_provider: Optional custom LLM provider extra_headers: Optional extra headers **kwargs: Additional parameters (e.g., include_image_base64, pages, image_limit) - + Returns: OCRResponse in Mistral OCR format with pages, model, usage_info, etc. - + Example: ```python import litellm - + # OCR with PDF response = await litellm.aocr( model="mistral/mistral-ocr-latest", @@ -64,7 +71,7 @@ async def aocr( }, include_image_base64=True ) - + # OCR with image response = await litellm.aocr( model="mistral/mistral-ocr-latest", @@ -73,7 +80,7 @@ async def aocr( "image_url": "https://example.com/image.png" } ) - + # OCR with base64 encoded PDF response = await litellm.aocr( model="mistral/mistral-ocr-latest", @@ -82,6 +89,12 @@ async def aocr( "document_url": f"data:application/pdf;base64,{base64_pdf}" } ) + + # OCR with local file + response = await litellm.aocr( + model="mistral/mistral-ocr-latest", + document={"type": "file", "file": "/path/to/document.pdf"} + ) ``` """ local_vars = locals() @@ -135,7 +148,7 @@ async def aocr( @client def ocr( model: str, - document: Dict[str, str], + document: Dict[str, Any], api_key: Optional[str] = None, api_base: Optional[str] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, @@ -145,26 +158,27 @@ def ocr( ) -> Union[OCRResponse, Coroutine[Any, Any, OCRResponse]]: """ Synchronous OCR function. - + Args: model: Model name (e.g., "mistral/mistral-ocr-latest") document: Document to process in Mistral format: - {"type": "document_url", "document_url": "https://..."} for PDFs/docs or - {"type": "image_url", "image_url": "https://..."} for images + {"type": "document_url", "document_url": "https://..."} for PDFs/docs, + {"type": "image_url", "image_url": "https://..."} for images, or + {"type": "file", "file": } for local files api_key: Optional API key api_base: Optional API base URL timeout: Optional timeout custom_llm_provider: Optional custom LLM provider extra_headers: Optional extra headers **kwargs: Additional parameters (e.g., include_image_base64, pages, image_limit) - + Returns: OCRResponse in Mistral OCR format with pages, model, usage_info, etc. - + Example: ```python import litellm - + # OCR with PDF response = litellm.ocr( model="mistral/mistral-ocr-latest", @@ -174,7 +188,7 @@ def ocr( }, include_image_base64=True ) - + # OCR with image response = litellm.ocr( model="mistral/mistral-ocr-latest", @@ -183,7 +197,7 @@ def ocr( "image_url": "https://example.com/image.png" } ) - + # OCR with base64 encoded PDF response = litellm.ocr( model="mistral/mistral-ocr-latest", @@ -192,7 +206,13 @@ def ocr( "document_url": f"data:application/pdf;base64,{base64_pdf}" } ) - + + # OCR with local file + response = litellm.ocr( + model="mistral/mistral-ocr-latest", + document={"type": "file", "file": "/path/to/document.pdf"} + ) + # Access pages for page in response.pages: print(f"Page {page.index}: {page.markdown}") @@ -203,24 +223,38 @@ def ocr( litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) _is_async = kwargs.pop("aocr", False) is True - - # Validate document parameter format (Mistral spec) - if not isinstance(document, dict): - raise ValueError(f"document must be a dict with 'type' and URL field, got {type(document)}") - - doc_type = document.get("type") - if doc_type not in ["document_url", "image_url"]: - raise ValueError(f"Invalid document type: {doc_type}. Must be 'document_url' or 'image_url'") - model, custom_llm_provider, dynamic_api_key, dynamic_api_base = ( - litellm.get_llm_provider( - model=model, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - api_key=api_key, + # Validate document parameter format + if not isinstance(document, dict): + raise ValueError( + f"document must be a dict with 'type' and URL/file field, got {type(document)}" ) + + doc_type = document.get("type") + + # Handle file type: convert to document_url/image_url with base64 data URI + if doc_type == "file": + document = convert_file_document_to_url_document(document) + doc_type = document.get("type") + + if doc_type not in ["document_url", "image_url"]: + raise ValueError( + f"Invalid document type: {doc_type}. " + "Must be 'document_url', 'image_url', or 'file'" + ) + + ( + model, + custom_llm_provider, + dynamic_api_key, + dynamic_api_base, + ) = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, ) - + # Update with dynamic values if available if dynamic_api_key: api_key = dynamic_api_key @@ -228,11 +262,11 @@ def ocr( api_base = dynamic_api_base # Get provider config - ocr_provider_config: Optional[BaseOCRConfig] = ( - ProviderConfigManager.get_provider_ocr_config( - model=model, - provider=litellm.LlmProviders(custom_llm_provider), - ) + ocr_provider_config: Optional[ + BaseOCRConfig + ] = ProviderConfigManager.get_provider_ocr_config( + model=model, + provider=litellm.LlmProviders(custom_llm_provider), ) if ocr_provider_config is None: @@ -246,21 +280,21 @@ def ocr( # Get litellm params using GenericLiteLLMParams (same as responses API) litellm_params = GenericLiteLLMParams(**kwargs) - + # Extract OCR-specific parameters from kwargs supported_params = ocr_provider_config.get_supported_ocr_params(model=model) non_default_params = {} for param in supported_params: if param in kwargs: non_default_params[param] = kwargs.pop(param) - + # Map parameters to provider-specific format optional_params = ocr_provider_config.map_ocr_params( non_default_params=non_default_params, optional_params={}, model=model, ) - + verbose_logger.debug(f"OCR optional_params after mapping: {optional_params}") # Pre Call logging @@ -300,3 +334,111 @@ def ocr( extra_kwargs=kwargs, ) + +################################################# +# Public utilities — used by the SDK and the proxy +################################################# + +_MIME_PATTERN = re.compile(r"^[\w.+-]+/[\w.+-]+$") + +_MIME_TYPE_MAP = { + ".pdf": "application/pdf", + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".tiff": "image/tiff", + ".tif": "image/tiff", + ".bmp": "image/bmp", +} + + +def get_mime_type(file_path: str) -> str: + """ + Determine MIME type from file path extension. + + Falls back to mimetypes.guess_type, then to 'application/octet-stream'. + """ + ext = os.path.splitext(file_path)[1].lower() + mime = _MIME_TYPE_MAP.get(ext) + if mime: + return mime + guessed, _ = mimetypes.guess_type(file_path) + return guessed or "application/octet-stream" + + +def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str, str]: + """ + Convert a file-type document dict to a document_url-type document dict + with an inline base64 data URI. + + Accepts document dicts like: + {"type": "file", "file": "/path/to/document.pdf"} # file path string + {"type": "file", "file": Path("/path/to/doc.pdf")} # pathlib.Path + {"type": "file", "file": } # file-like object (BinaryIO) + {"type": "file", "file": b"raw bytes"} # raw bytes + + Returns: + {"type": "document_url", "document_url": "data:;base64,"} + or {"type": "image_url", "image_url": "data:;base64,"} + """ + file_input = document.get("file") + if file_input is None: + raise ValueError( + "document with type='file' must include a 'file' field containing " + "a file path (str), pathlib.Path, file-like object, or bytes" + ) + + file_bytes: bytes + mime_type: str = "application/octet-stream" + file_name: Optional[str] = None + + if isinstance(file_input, (str, Path)): + file_path = str(file_input) + if not os.path.isfile(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + mime_type = get_mime_type(file_path) + file_name = os.path.basename(file_path) + with open(file_path, "rb") as f: + file_bytes = f.read() + elif isinstance(file_input, bytes): + file_bytes = file_input + elif isinstance(file_input, IOBase) or hasattr(file_input, "read"): + if hasattr(file_input, "name"): + file_name = getattr(file_input, "name", None) + if file_name: + mime_type = get_mime_type(file_name) + file_bytes = file_input.read() + if isinstance(file_bytes, str): + file_bytes = file_bytes.encode("utf-8") + else: + raise ValueError( + f"Unsupported file input type: {type(file_input)}. " + "Expected str (file path), pathlib.Path, bytes, or a file-like object." + ) + + if not file_bytes: + raise ValueError("File is empty or could not be read") + + if "mime_type" in document: + mime_type = document["mime_type"] + + if not _MIME_PATTERN.match(mime_type): + raise ValueError(f"Invalid MIME type: {mime_type}") + + base64_data = base64.b64encode(file_bytes).decode("utf-8") + data_uri = f"data:{mime_type};base64,{base64_data}" + + if mime_type.startswith("image/"): + verbose_logger.debug( + f"OCR file input: Converted file to image_url data URI " + f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + ) + return {"type": "image_url", "image_url": data_uri} + else: + verbose_logger.debug( + f"OCR file input: Converted file to document_url data URI " + f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + ) + return {"type": "document_url", "document_url": data_uri} diff --git a/litellm/proxy/ocr_endpoints/endpoints.py b/litellm/proxy/ocr_endpoints/endpoints.py index c1092a06b48..4f31c762df1 100644 --- a/litellm/proxy/ocr_endpoints/endpoints.py +++ b/litellm/proxy/ocr_endpoints/endpoints.py @@ -1,9 +1,14 @@ #### OCR Endpoints ##### +import json +from typing import Any, Dict, Optional, cast + import orjson -from fastapi import APIRouter, Depends, Request, Response +from fastapi import APIRouter, Depends, Request, Response, UploadFile from fastapi.responses import ORJSONResponse +from litellm._logging import verbose_proxy_logger +from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -11,6 +16,171 @@ from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessin router = APIRouter() +def _build_document_from_upload( + file_content: bytes, + filename: Optional[str], + content_type: Optional[str], +) -> Dict[str, str]: + """ + Convert uploaded file bytes into a Mistral-format document dict with base64 data URI. + + Delegates to convert_file_document_to_url_document after resolving MIME type + from the upload's content_type header or filename. + """ + mime_type = content_type.split(";")[0].strip() if content_type else None + if not mime_type or mime_type == "application/octet-stream": + if filename: + mime_type = get_mime_type(filename) + + return convert_file_document_to_url_document( + { + "type": "file", + "file": file_content, + "mime_type": mime_type or "application/octet-stream", + } + ) + + +async def _parse_multipart_form(request: Request) -> Dict[str, Any]: + """ + Extract OCR data from a multipart form request. + + Uses the cached form if already parsed by auth middleware, + otherwise parses the form from the request. + + Returns: + A dict with 'document', 'model', and any other OCR params. + """ + try: + form = await request.form() + except Exception as e: + raise ValueError( + f"Failed to parse multipart form data: {str(e)}. " + "When using curl with --form/-F, do NOT set the Content-Type header " + "manually — curl will set it automatically with the required boundary." + ) + + uploaded_file = form.get("file") + # request.form() may return either a FastAPI or Starlette UploadFile + # depending on middleware; check both via isinstance (FastAPI's UploadFile + # is a subclass of Starlette's) and fall back to duck-type check. + if uploaded_file is None or ( + not isinstance(uploaded_file, UploadFile) and not hasattr(uploaded_file, "read") + ): + raise ValueError( + "Multipart OCR request must include a 'file' field with the document to process" + ) + + uploaded_file = cast(UploadFile, uploaded_file) + + # Seek to start in case the file was already partially read by middleware + await uploaded_file.seek(0) + file_content = await uploaded_file.read() + if not file_content: + raise ValueError("Uploaded file is empty") + + document = _build_document_from_upload( + file_content=file_content, + filename=uploaded_file.filename, + content_type=uploaded_file.content_type, + ) + + data: Dict[str, Any] = {"document": document} + + for field_name, field_value in form.items(): + if field_name in ("file", "document"): + continue + # Try to parse JSON values (e.g. pages=[0,1,2]) + if isinstance(field_value, str): + try: + data[field_name] = json.loads(field_value) + except (json.JSONDecodeError, ValueError): + data[field_name] = field_value + else: + data[field_name] = field_value + + verbose_proxy_logger.debug( + f"OCR multipart form request parsed - model: {data.get('model')}, " + f"document_type: {document['type']}, " + f"filename: {uploaded_file.filename}" + ) + + return data + + +async def _parse_ocr_request(request: Request) -> Dict[str, Any]: + """ + Parse an OCR request, supporting both JSON and multipart form data. + + JSON body (existing behavior): + { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "document_url", "document_url": "https://..."} + } + + Multipart form data (new): + - file: the uploaded file + - model: model name (form field) + - Any other OCR params as form fields (pages, include_image_base64, etc.) + + Returns: + A dict suitable for passing to the OCR processing pipeline. + """ + content_type = request.headers.get("content-type", "") + + if "multipart/form-data" in content_type.lower(): + return await _parse_multipart_form(request) + + # --- JSON body (existing behavior) --- + try: + body = await request.body() + except RuntimeError: + # Body stream was consumed by auth middleware (e.g., form parsing). + body = b"" + + if not body: + # The body may be empty because the auth middleware already parsed + # it as form data (e.g., _read_request_body called request.form()). + # Check if form data is available. + if getattr(request, "_form", None) is not None: + verbose_proxy_logger.debug( + "OCR request body is empty but form data is available from middleware — " + "processing as multipart form." + ) + return await _parse_multipart_form(request) + + raise ValueError( + "Empty request body. For file uploads, use multipart/form-data content type " + "with a file field. When using curl with --form/-F, do NOT set the Content-Type " + "header manually." + ) + + try: + data = orjson.loads(body) + except orjson.JSONDecodeError as e: + raise ValueError( + f"Invalid JSON in request body: {e}. " + "Ensure the request body is valid JSON with Content-Type: application/json, " + "or use multipart/form-data for file uploads." + ) + + # Security: reject type="file" documents received via JSON. + # The "file" document type is designed for local SDK usage where the + # caller and the process share a filesystem. In the proxy context the + # caller is remote, so allowing a file-path string would let an + # authenticated user read arbitrary files from the server's filesystem. + # File uploads must go through multipart/form-data instead. + doc = data.get("document") if isinstance(data, dict) else None + if isinstance(doc, dict) and doc.get("type") == "file": + raise ValueError( + "document type 'file' is not supported through the JSON API. " + "To upload a local file, use multipart/form-data with a 'file' field. " + "For JSON requests, use 'document_url' or 'image_url' document types." + ) + + return data + + @router.post( "/v1/ocr", dependencies=[Depends(user_api_key_auth)], @@ -30,23 +200,30 @@ async def ocr( ): """ OCR endpoint for extracting text from documents and images. - - Follows the Mistral OCR API spec: - https://docs.mistral.ai/capabilities/vision/#optical-character-recognition-ocr - - Example: + + Supports two input modes: + + **1. JSON body** (Mistral OCR API compatible): ```bash curl -X POST "http://localhost:4000/v1/ocr" \ -H "Authorization: Bearer sk-1234" \ -H "Content-Type: application/json" \ -d '{ - "model": "mistral/mistral-ocr-latest", + "model": "mistral-ocr", "document": { "type": "document_url", "document_url": "https://arxiv.org/pdf/2201.04234" } }' ``` + + **2. Multipart form file upload**: + ```bash + curl -X POST "http://localhost:4000/v1/ocr" \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@document.pdf" + ``` """ from litellm.proxy.proxy_server import ( general_settings, @@ -62,13 +239,14 @@ async def ocr( version, ) - # Read request body - body = await request.body() - data = orjson.loads(body) - - # Process request using ProxyBaseLLMRequestProcessing - processor = ProxyBaseLLMRequestProcessing(data=data) + data: dict = {} try: + # Parse request body (JSON or multipart form) + data = await _parse_ocr_request(request) + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + return await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, @@ -88,10 +266,10 @@ async def ocr( version=version, ) except Exception as e: + processor = ProxyBaseLLMRequestProcessing(data=data) raise await processor._handle_llm_api_exception( e=e, user_api_key_dict=user_api_key_dict, proxy_logging_obj=proxy_logging_obj, version=version, ) - diff --git a/tests/test_litellm/ocr/__init__.py b/tests/test_litellm/ocr/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/ocr/test_ocr_file_input.py b/tests/test_litellm/ocr/test_ocr_file_input.py new file mode 100644 index 00000000000..492253e2f11 --- /dev/null +++ b/tests/test_litellm/ocr/test_ocr_file_input.py @@ -0,0 +1,464 @@ +""" +Tests for OCR file input support. + +Tests that: +1. The SDK document parameter with type="file" correctly converts file paths, + file objects, and raw bytes to base64 data URIs before sending to providers. +2. The proxy _build_document_from_upload helper correctly handles uploaded file bytes. +3. The proxy rejects type="file" documents received via JSON (security guard). +4. The proxy returns user-friendly errors for invalid JSON bodies. +""" +import base64 +import os +import tempfile +from io import BytesIO +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import orjson +import pytest + +from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type + + +class TestGetMimeType: + def test_should_detect_pdf_mime_type(self): + assert get_mime_type("document.pdf") == "application/pdf" + + def test_should_detect_png_mime_type(self): + assert get_mime_type("image.png") == "image/png" + + def test_should_detect_jpg_mime_type(self): + assert get_mime_type("photo.jpg") == "image/jpeg" + + def test_should_detect_jpeg_mime_type(self): + assert get_mime_type("photo.jpeg") == "image/jpeg" + + def test_should_detect_gif_mime_type(self): + assert get_mime_type("animation.gif") == "image/gif" + + def test_should_detect_webp_mime_type(self): + assert get_mime_type("image.webp") == "image/webp" + + def test_should_detect_tiff_mime_type(self): + assert get_mime_type("scan.tiff") == "image/tiff" + + def test_should_detect_tif_mime_type(self): + assert get_mime_type("scan.tif") == "image/tiff" + + def test_should_detect_bmp_mime_type(self): + assert get_mime_type("bitmap.bmp") == "image/bmp" + + def test_should_be_case_insensitive(self): + assert get_mime_type("DOCUMENT.PDF") == "application/pdf" + assert get_mime_type("IMAGE.PNG") == "image/png" + + def test_should_fallback_for_unknown_extension(self): + result = get_mime_type("file.xyz123") + assert isinstance(result, str) + + +class TestConvertFileDocumentToUrlDocument: + def test_should_convert_pdf_file_path_to_document_url(self): + """File path to a PDF should produce type=document_url with base64 data URI.""" + pdf_content = b"%PDF-1.4 test content" + + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(pdf_content) + f.flush() + tmp_path = f.name + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == pdf_content + finally: + os.unlink(tmp_path) + + def test_should_convert_image_file_path_to_image_url(self): + """File path to a PNG image should produce type=image_url with base64 data URI.""" + png_content = b"\x89PNG\r\n\x1a\n fake png content" + + with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f: + f.write(png_content) + f.flush() + tmp_path = f.name + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + b64_data = result["image_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == png_content + finally: + os.unlink(tmp_path) + + def test_should_convert_pathlib_path(self): + """pathlib.Path objects should work the same as string paths.""" + content = b"test pdf content" + + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(content) + f.flush() + tmp_path = Path(f.name) + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + finally: + os.unlink(str(tmp_path)) + + def test_should_convert_raw_bytes(self): + """Raw bytes should be converted using a fallback MIME type.""" + content = b"raw bytes content" + + result = convert_file_document_to_url_document( + {"type": "file", "file": content} + ) + + assert result["type"] == "document_url" + assert "base64," in result["document_url"] + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == content + + def test_should_convert_raw_bytes_with_explicit_mime_type(self): + """Raw bytes with explicit mime_type should use the specified MIME type.""" + content = b"raw pdf content" + + result = convert_file_document_to_url_document( + {"type": "file", "file": content, "mime_type": "application/pdf"} + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + def test_should_convert_raw_bytes_with_image_mime_type(self): + """Raw bytes with an image MIME type should produce type=image_url.""" + content = b"raw image content" + + result = convert_file_document_to_url_document( + {"type": "file", "file": content, "mime_type": "image/jpeg"} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/jpeg;base64,") + + def test_should_convert_file_like_object(self): + """BytesIO and other file-like objects should be supported.""" + content = b"file-like content" + file_obj = BytesIO(content) + + result = convert_file_document_to_url_document( + {"type": "file", "file": file_obj} + ) + + assert result["type"] == "document_url" + assert "base64," in result["document_url"] + + def test_should_convert_file_like_object_with_name(self): + """File-like objects with a .name attribute should detect MIME from the name.""" + content = b"file-like png content" + file_obj = BytesIO(content) + file_obj.name = "test_image.png" + + result = convert_file_document_to_url_document( + {"type": "file", "file": file_obj} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + def test_should_raise_error_for_missing_file_field(self): + """Missing 'file' field should raise ValueError.""" + with pytest.raises(ValueError, match="must include a 'file' field"): + convert_file_document_to_url_document({"type": "file"}) + + def test_should_raise_error_for_nonexistent_file_path(self): + """Non-existent file path should raise FileNotFoundError.""" + with pytest.raises(FileNotFoundError, match="File not found"): + convert_file_document_to_url_document( + {"type": "file", "file": "/nonexistent/path/to/file.pdf"} + ) + + def test_should_raise_error_for_empty_file(self): + """Empty file should raise ValueError.""" + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + tmp_path = f.name + + try: + with pytest.raises(ValueError, match="File is empty"): + convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + finally: + os.unlink(tmp_path) + + def test_should_raise_error_for_unsupported_type(self): + """Unsupported file input types should raise ValueError.""" + with pytest.raises(ValueError, match="Unsupported file input type"): + convert_file_document_to_url_document({"type": "file", "file": 12345}) + + def test_should_raise_error_for_invalid_mime_type(self): + """MIME types with special characters should be rejected.""" + content = b"some content" + with pytest.raises(ValueError, match="Invalid MIME type"): + convert_file_document_to_url_document( + {"type": "file", "file": content, "mime_type": "text/html; charset=utf-8\nX-Injected: true"} + ) + + def test_should_override_mime_type_for_file_path(self): + """Explicit mime_type should override auto-detection from extension.""" + content = b"some content" + + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(content) + f.flush() + tmp_path = f.name + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path, "mime_type": "image/png"} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + finally: + os.unlink(tmp_path) + + +class TestBuildDocumentFromUpload: + """Test the proxy endpoint's file upload to document conversion helper.""" + + @pytest.fixture(autouse=True) + def _import_helper(self): + """Import the proxy helper, skip if proxy deps aren't installed.""" + try: + from litellm.proxy.ocr_endpoints.endpoints import ( + _build_document_from_upload, + ) + + self._build = _build_document_from_upload + except ImportError: + pytest.skip("Proxy dependencies (fastapi/orjson) not installed") + + def test_should_build_document_url_for_pdf(self): + content = b"%PDF-1.4 test content" + + result = self._build( + file_content=content, + filename="document.pdf", + content_type="application/pdf", + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == content + + def test_should_build_image_url_for_png(self): + content = b"\x89PNG fake png" + + result = self._build( + file_content=content, + filename="screenshot.png", + content_type="image/png", + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + def test_should_build_image_url_for_jpeg(self): + content = b"\xff\xd8\xff fake jpeg" + + result = self._build( + file_content=content, + filename="photo.jpg", + content_type="image/jpeg", + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/jpeg;base64,") + + def test_should_detect_mime_from_filename_when_content_type_is_octet_stream(self): + content = b"pdf content" + + result = self._build( + file_content=content, + filename="report.pdf", + content_type="application/octet-stream", + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + def test_should_detect_mime_from_filename_when_content_type_is_none(self): + content = b"png content" + + result = self._build( + file_content=content, + filename="image.png", + content_type=None, + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + def test_should_fallback_to_octet_stream_for_unknown(self): + content = b"unknown content" + + result = self._build( + file_content=content, + filename=None, + content_type=None, + ) + + assert result["type"] == "document_url" + assert "application/octet-stream" in result["document_url"] + + def test_should_preserve_base64_content_correctly(self): + content = b"Hello, World! \x00\x01\x02\xff" + + result = self._build( + file_content=content, + filename="test.pdf", + content_type="application/pdf", + ) + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == content + + def test_should_strip_mime_parameters_from_content_type(self): + """Content-Type with parameters (e.g. charset) should be stripped to the base MIME type.""" + content = b"%PDF-1.4 test" + + result = self._build( + file_content=content, + filename="doc.pdf", + content_type="application/pdf; charset=utf-8", + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + def test_should_strip_mime_parameters_with_multiple_params(self): + """Content-Type with multiple parameters should still be stripped correctly.""" + content = b"image data" + + result = self._build( + file_content=content, + filename="img.png", + content_type="image/png; charset=utf-8; boundary=something", + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + +class TestProxySecurityGuard: + """Test that the proxy rejects type='file' documents in JSON requests + and that multipart form fields cannot override the constructed document.""" + + @pytest.fixture(autouse=True) + def _import_helpers(self): + """Import the proxy helpers, skip if proxy deps aren't installed.""" + try: + from litellm.proxy.ocr_endpoints.endpoints import ( + _parse_multipart_form, + _parse_ocr_request, + ) + + self._parse = _parse_ocr_request + self._parse_multipart = _parse_multipart_form + except ImportError: + pytest.skip("Proxy dependencies (fastapi/orjson) not installed") + + @pytest.mark.asyncio + async def test_should_reject_file_type_document_in_json_body(self): + """type='file' in a JSON body must be rejected to prevent server-side file reads.""" + body = orjson.dumps( + { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "file", "file": "/etc/passwd"}, + } + ) + + mock_request = MagicMock() + mock_request.headers = {"content-type": "application/json"} + mock_request.body = AsyncMock(return_value=body) + mock_request._form = None + + with pytest.raises(ValueError, match="not supported through the JSON API"): + await self._parse(mock_request) + + @pytest.mark.asyncio + async def test_should_accept_document_url_type_in_json_body(self): + """type='document_url' in a JSON body should pass through normally.""" + expected = { + "model": "mistral/mistral-ocr-latest", + "document": { + "type": "document_url", + "document_url": "https://example.com/doc.pdf", + }, + } + body = orjson.dumps(expected) + + mock_request = MagicMock() + mock_request.headers = {"content-type": "application/json"} + mock_request.body = AsyncMock(return_value=body) + mock_request._form = None + + result = await self._parse(mock_request) + assert result["document"]["type"] == "document_url" + + @pytest.mark.asyncio + async def test_should_raise_on_invalid_json_body(self): + """Invalid JSON should produce a user-friendly ValueError.""" + mock_request = MagicMock() + mock_request.headers = {"content-type": "application/json"} + mock_request.body = AsyncMock(return_value=b"not valid json{{{") + mock_request._form = None + + with pytest.raises(ValueError, match="Invalid JSON in request body"): + await self._parse(mock_request) + + @pytest.mark.asyncio + async def test_should_ignore_document_form_field_injection(self): + """A 'document' form field must not override the document built from the uploaded file.""" + from starlette.datastructures import UploadFile + + file_content = b"%PDF-1.4 legit content" + upload = UploadFile(filename="legit.pdf", file=BytesIO(file_content)) + + injected = '{"type": "file", "file": "/etc/passwd"}' + + mock_form = { + "file": upload, + "model": "mistral/mistral-ocr-latest", + "document": injected, + } + + mock_request = MagicMock() + mock_request.headers = {"content-type": "multipart/form-data; boundary=---"} + mock_request.form = AsyncMock(return_value=mock_form) + + result = await self._parse_multipart(mock_request) + + assert result["document"]["type"] == "document_url" + assert result["document"]["document_url"].startswith("data:application/pdf;base64,") + assert result["model"] == "mistral/mistral-ocr-latest" From 1144d05cbab94c473964ab8be575015f767cd073 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 15:57:56 -0300 Subject: [PATCH 48/54] feat(models): add gpt-audio-1.5 to model cost map New OpenAI audio model released 2026-02-23. Adds pricing and capability metadata for gpt-audio-1.5 (128K context, 16K output, audio I/O). Closes #22269 --- model_prices_and_context_window.json | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e4d7a6a02f2..e9002084de9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -19085,6 +19085,42 @@ "supports_tool_choice": true, "supports_vision": false }, + "gpt-audio-1.5": { + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/realtime", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "gpt-audio-2025-08-28": { "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, From 29bb73ffcace33e65f39429493a10b1ecb76c909 Mon Sep 17 00:00:00 2001 From: Gaurav Singh <103016722+gavksingh@users.noreply.github.com> Date: Fri, 27 Feb 2026 13:59:08 -0500 Subject: [PATCH 49/54] fix(mcp): strip stale mcp-session-id header to prevent 400 in multi-worker deployments (#20992) (#21417) In a multi-worker Uvicorn setup, a client that reconnects to a different worker sends an mcp-session-id that the new worker has never seen. The MCP SDK returns 400 because the session is unknown. Fix: add _handle_stale_mcp_session() which inspects the inbound mcp-session-id header before the request reaches the SDK. If the session is not in this worker's _server_instances: - Non-DELETE: strip the header so the SDK creates a fresh session - DELETE: return 200 immediately (idempotent, session already gone) No new dependencies, no Redis, no latency added to the hot path. Fixes https://github.com/BerriAI/litellm/issues/20992 --- .../proxy/_experimental/mcp_server/server.py | 99 ++++++++++++------- tests/mcp_tests/test_mcp_server.py | 3 +- 2 files changed, 64 insertions(+), 38 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 48c837c1e4f..5b3d5bd60e2 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -5,6 +5,7 @@ LiteLLM MCP Server Routes import asyncio import contextlib + import traceback import uuid from datetime import datetime @@ -84,6 +85,7 @@ except ImportError as e: _SESSION_MANAGERS_INITIALIZED = False _INITIALIZATION_LOCK = asyncio.Lock() + if MCP_AVAILABLE: from mcp.server import Server @@ -1919,65 +1921,86 @@ if MCP_AVAILABLE: mgr: "StreamableHTTPSessionManager", ) -> bool: """ - Handle stale MCP session IDs to prevent "Session not found" errors. - - When clients reconnect after a server restart or session cleanup, they may - send a session ID that no longer exists. This function handles two scenarios: - - 1. Non-DELETE requests: Strip the stale session ID header so the session - manager creates a fresh session transparently. - - 2. DELETE requests: Return success (200) immediately for idempotent behavior, - since the desired state (session doesn't exist) is already achieved. + Inspect the incoming ``mcp-session-id`` header **before** the + request reaches the MCP SDK. If the session is stale (not known + to this worker), strip the header so the SDK creates a fresh + stateless session instead of returning a 400. Returns: - True if the request was handled (DELETE on non-existent session) - False if the request should continue to the session manager + True if the request was fully handled (e.g. DELETE on + non-existent session). False if the request should continue + to the session manager. - Fixes https://github.com/BerriAI/litellm/issues/20292 + Fixes https://github.com/BerriAI/litellm/issues/20992 """ _mcp_session_header = b"mcp-session-id" + _headers = scope.get("headers", []) + + def _normalize_header_name(header_name: Any) -> Optional[bytes]: + if isinstance(header_name, bytes): + return header_name.lower() + if isinstance(header_name, str): + return header_name.lower().encode("utf-8", errors="replace") + return None + _session_id: Optional[str] = None - for header_name, header_value in scope.get("headers", []): - if header_name == _mcp_session_header: - _session_id = header_value.decode("utf-8", errors="replace") + for header_name, header_value in _headers: + if _normalize_header_name(header_name) == _mcp_session_header: + if isinstance(header_value, bytes): + _session_id = header_value.decode("utf-8", errors="replace") + else: + _session_id = str(header_value) break if _session_id is None: return False + # Check in-memory session tracking known_sessions = getattr(mgr, "_server_instances", None) - if known_sessions is None or _session_id in known_sessions: - # Session exists or we can't check - let the session manager handle it + # If we cannot inspect known_sessions, let the manager handle it + if known_sessions is None: return False - # Session doesn't exist - handle based on request method + # If session exists in this worker's memory, let the manager handle it + try: + if _session_id in known_sessions: + return False + except Exception: + verbose_logger.debug( + "Unable to inspect active MCP sessions for '%s'. " + "Deferring to session manager.", + _session_id, + ) + return False + + # --- Session not in this worker's memory --- method = scope.get("method", "").upper() - + if method == "DELETE": - # Idempotent DELETE: session doesn't exist, return success verbose_logger.info( - f"DELETE request for non-existent MCP session '{_session_id}'. " - "Returning success (idempotent DELETE)." + "DELETE request for non-existent MCP session '%s'. " + "Returning success (idempotent DELETE).", + _session_id, ) success_response = JSONResponse( status_code=200, - content={"message": "Session terminated successfully"} + content={"message": "Session terminated successfully"}, ) await success_response(scope, receive, send) return True - else: - # Non-DELETE: strip stale session ID to allow new session creation - verbose_logger.warning( - "MCP session ID '%s' not found in active sessions. " - "Stripping stale header to force new session creation.", - _session_id, - ) - scope["headers"] = [ - (k, v) for k, v in scope["headers"] - if k != _mcp_session_header - ] - return False + + # Non-DELETE: strip stale session ID to allow new session creation + verbose_logger.warning( + "MCP session ID '%s' not found in this worker's memory. " + "Stripping stale header to force new session creation.", + _session_id, + ) + scope["headers"] = [ + (k, v) + for k, v in _headers + if _normalize_header_name(k) != _mcp_session_header + ] + return False async def handle_streamable_http_mcp( scope: Scope, receive: Receive, send: Send @@ -2055,7 +2078,9 @@ if MCP_AVAILABLE: # Handle stale session IDs - either strip them for reconnection # or return success for idempotent DELETE operations - handled = await _handle_stale_mcp_session(scope, receive, send, session_manager) + handled = await _handle_stale_mcp_session( + scope, receive, send, session_manager + ) if handled: # Request was fully handled (e.g., DELETE on non-existent session) return diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 930b0a03042..dc1e2068365 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -427,7 +427,8 @@ async def test_streamable_http_mcp_handler_mock(): # Call the handler await handle_streamable_http_mcp(mock_scope, mock_receive, mock_send) - # Verify session manager handle_request was called + # Verify session manager handle_request was called with correct args + # send is passed directly (no wrapper) mock_session_manager.handle_request.assert_called_once_with( mock_scope, mock_receive, mock_send ) From 1f887547f6652d7bc5f87fa44f8563131b853124 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 16:01:09 -0300 Subject: [PATCH 50/54] feat(models): add gpt-realtime-1.5 to model cost map New OpenAI realtime model released 2026-02-23. Adds pricing and capability metadata for gpt-realtime-1.5 (32K context, 4K output, audio/image/text I/O). Unlike gpt-realtime, this model also supports Chat Completions and Responses endpoints (not just WebSocket). Closes #22266 --- model_prices_and_context_window.json | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e4d7a6a02f2..041f76af70b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20802,6 +20802,40 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "gpt-realtime-1.5": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1.6e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, From c7ab631bb045caebc588b94a890305612d47146e Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 16:06:10 -0300 Subject: [PATCH 51/54] fix(audio): detect gpt-realtime models as audio-capable for Chat Completions gpt-realtime-1.5 supports Chat Completions with audio params but is_model_gpt_audio_model only checked for "audio" in the model name. Add "realtime" check so the audio parameter is passed through correctly. --- litellm/llms/openai/chat/gpt_audio_transformation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/chat/gpt_audio_transformation.py b/litellm/llms/openai/chat/gpt_audio_transformation.py index 581ffea2db4..66270b1da9d 100644 --- a/litellm/llms/openai/chat/gpt_audio_transformation.py +++ b/litellm/llms/openai/chat/gpt_audio_transformation.py @@ -29,7 +29,9 @@ class OpenAIGPTAudioConfig(OpenAIGPTConfig): return all_openai_params + audio_specific_params def is_model_gpt_audio_model(self, model: str) -> bool: - if model in litellm.open_ai_chat_completion_models and "audio" in model: + if model in litellm.open_ai_chat_completion_models and ( + "audio" in model or "realtime" in model + ): return True return False From da73e54b1bd9e8c8974e912500b989ba080f1c02 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 16:10:01 -0300 Subject: [PATCH 52/54] fix: gpt-realtime-1.5 only supports /v1/realtime endpoint Remove /v1/chat/completions and /v1/responses from supported_endpoints and revert the audio model detection change since gpt-realtime-1.5 does not go through Chat Completions. --- litellm/llms/openai/chat/gpt_audio_transformation.py | 4 +--- model_prices_and_context_window.json | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_audio_transformation.py b/litellm/llms/openai/chat/gpt_audio_transformation.py index 66270b1da9d..581ffea2db4 100644 --- a/litellm/llms/openai/chat/gpt_audio_transformation.py +++ b/litellm/llms/openai/chat/gpt_audio_transformation.py @@ -29,9 +29,7 @@ class OpenAIGPTAudioConfig(OpenAIGPTConfig): return all_openai_params + audio_specific_params def is_model_gpt_audio_model(self, model: str) -> bool: - if model in litellm.open_ai_chat_completion_models and ( - "audio" in model or "realtime" in model - ): + if model in litellm.open_ai_chat_completion_models and "audio" in model: return True return False diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 041f76af70b..66d5f8b7c35 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20816,8 +20816,6 @@ "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses", "/v1/realtime" ], "supported_modalities": [ From 1ca4dd8542bfcd3fc2ec691b6125d78975c546e2 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 16:11:15 -0300 Subject: [PATCH 53/54] fix: gpt-audio-1.5 only supports /v1/chat/completions endpoint --- model_prices_and_context_window.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e9002084de9..35b820c1a6e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -19096,10 +19096,7 @@ "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses", - "/v1/realtime", - "/v1/batch" + "/v1/chat/completions" ], "supported_modalities": [ "text", From ad9c70ec5d14e8947a33d796b9c0b85ae32b1b16 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Fri, 27 Feb 2026 11:14:13 -0800 Subject: [PATCH 54/54] Add LLMClientCache regression tests for httpx client eviction safety MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regression tests for PR #22247 — ensures cache eviction (capacity and TTL) does not close httpx clients that are still in use. --- .../caching/test_redis_connection_pool.py | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/tests/test_litellm/caching/test_redis_connection_pool.py b/tests/test_litellm/caching/test_redis_connection_pool.py index f6e429ceff9..3d808438850 100644 --- a/tests/test_litellm/caching/test_redis_connection_pool.py +++ b/tests/test_litellm/caching/test_redis_connection_pool.py @@ -1,15 +1,13 @@ -""" -Regression tests for Redis connection pool leak fixes (RC1-RC5). - -Tests are pure unit tests — no Redis server required. -""" +"""Redis connection pool and LLMClientCache eviction tests.""" from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import redis.asyncio as async_redis from litellm._redis import get_redis_async_client, get_redis_connection_pool +from litellm.caching.llm_caching_handler import LLMClientCache def test_url_config_uses_passed_pool(): @@ -129,3 +127,33 @@ async def test_disconnect_idempotent(): await cache.disconnect() # should not raise +# Regression: cache eviction must not close shared httpx clients (PR #22247) + +@pytest.mark.asyncio +async def test_httpx_client_survives_capacity_eviction(): + """Evicting an httpx client from LLMClientCache must NOT close it.""" + cache = LLMClientCache(max_size_in_memory=1, default_ttl=600) + client = httpx.AsyncClient() + + cache.set_cache("client_1", client) + # Exceed capacity — client_1 gets evicted + cache.set_cache("client_2", "other") + + assert not client.is_closed + await client.aclose() + + +@pytest.mark.asyncio +async def test_httpx_client_survives_ttl_eviction(): + """Evicting an httpx client via TTL expiry must NOT close it.""" + cache = LLMClientCache(max_size_in_memory=200, default_ttl=600) + client = httpx.AsyncClient() + + # TTL=0 so it expires immediately + cache.set_cache("client_1", client, ttl=0) + cache.evict_cache() + + assert not client.is_closed + await client.aclose() + +