From 1f83aafe3befe4438f73ebeaea22409cabc8bdbb Mon Sep 17 00:00:00 2001 From: Chesars Date: Sat, 31 Jan 2026 00:31:49 -0300 Subject: [PATCH 01/46] Add deprecation dates for xAI grok-2-vision-1212 and grok-3-mini models Per xAI announcement, these models will be deprecated on February 28, 2026: - xai/grok-2-vision-1212 - xai/grok-3-mini - xai/grok-3-mini-beta --- model_prices_and_context_window.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d958ea4503a..812e62c056a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -30354,6 +30354,7 @@ "supports_web_search": true }, "xai/grok-2-vision-1212": { + "deprecation_date": "2026-02-28", "input_cost_per_image": 2e-06, "input_cost_per_token": 2e-06, "litellm_provider": "xai", @@ -30452,6 +30453,7 @@ "supports_web_search": true }, "xai/grok-3-mini": { + "deprecation_date": "2026-02-28", "input_cost_per_token": 3e-07, "litellm_provider": "xai", "max_input_tokens": 131072, @@ -30467,6 +30469,7 @@ "supports_web_search": true }, "xai/grok-3-mini-beta": { + "deprecation_date": "2026-02-28", "input_cost_per_token": 3e-07, "litellm_provider": "xai", "max_input_tokens": 131072, From b417de6fe8ed389039e97a8807386cf48c34f934 Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 5 Feb 2026 17:44:01 -0300 Subject: [PATCH 02/46] feat(model_prices): add OpenRouter native models to model cost map Add openrouter/auto, openrouter/free, and openrouter/bodybuilder to the model prices and context window JSON. These are OpenRouter's own native models with their capabilities (context window, function calling, vision, reasoning, etc). --- model_prices_and_context_window.json | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 66c8e55c3ae..8803e6e7df6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24820,6 +24820,41 @@ "supports_prompt_caching": false, "supports_computer_use": false }, + "openrouter/openrouter/auto": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "openrouter", + "max_input_tokens": 2000000, + "max_tokens": 2000000, + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_audio_input": true + }, + "openrouter/openrouter/free": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_tokens": 200000, + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/openrouter/bodybuilder": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "chat" + }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", From b7b48fd8ba789b4b60c5030890113b04e37e83e2 Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 5 Feb 2026 18:04:54 -0300 Subject: [PATCH 03/46] add supports_video_input for openrouter/auto --- model_prices_and_context_window.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8803e6e7df6..48ac68287b7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24832,7 +24832,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_vision": true, - "supports_audio_input": true + "supports_audio_input": true, + "supports_video_input": true }, "openrouter/openrouter/free": { "input_cost_per_token": 0, From 1d0c2f47a7b2291beb7fd7a45fa342f80ee455e0 Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 5 Feb 2026 18:33:02 -0300 Subject: [PATCH 04/46] docs: add Opus 4.6 to OpenRouter model map and update Anthropic docs - Add openrouter/anthropic/claude-opus-4.6 to model cost map - Update Bedrock docs to list Opus 4.6 and Sonnet 4.5 as 1M context compatible - Add missing Opus 4.6, Opus 4.5, and Opus 4.1 to Anthropic supported models table --- docs/my-website/docs/providers/anthropic.md | 4 ++++ docs/my-website/docs/providers/bedrock.md | 4 ++-- ...odel_prices_and_context_window_backup.json | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 446d663c5ac..ee04faaf7b3 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -4,6 +4,7 @@ import TabItem from '@theme/TabItem'; # Anthropic LiteLLM supports all anthropic models. +- `claude-opus-4-6-20260205` - `claude-sonnet-4-5-20250929` - `claude-opus-4-5-20251101` - `claude-opus-4-1-20250805` @@ -415,7 +416,10 @@ print(response) | Model Name | Function Call | |------------------|--------------------------------------------| +| claude-opus-4-6 | `completion('claude-opus-4-6-20260205', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-sonnet-4-5 | `completion('claude-sonnet-4-5-20250929', messages)` | `os.environ['ANTHROPIC_API_KEY']` | +| claude-opus-4-5 | `completion('claude-opus-4-5-20251101', messages)` | `os.environ['ANTHROPIC_API_KEY']` | +| claude-opus-4-1 | `completion('claude-opus-4-1-20250805', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-opus-4 | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-sonnet-4 | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-3.7 | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']` | diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index e546ed97656..bb07216a295 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -660,7 +660,7 @@ Same as [Anthropic API response](../providers/anthropic#usage---thinking--reason LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic-beta` header. This enables access to experimental features like: -- **1M Context Window** - Up to 1 million tokens of context (Claude Sonnet 4) +- **1M Context Window** - Up to 1 million tokens of context (Claude Opus 4.6, Sonnet 4.5, Sonnet 4) - **Computer Use Tools** - AI that can interact with computer interfaces - **Token-Efficient Tools** - More efficient tool usage patterns - **Extended Output** - Up to 128K output tokens @@ -670,7 +670,7 @@ LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic | Beta Feature | Header Value | Compatible Models | Description | |--------------|-------------|------------------|-------------| -| 1M Context Window | `context-1m-2025-08-07` | Claude Sonnet 4 | Enable 1 million token context window | +| 1M Context Window | `context-1m-2025-08-07` | Claude Opus 4.6, Sonnet 4.5, Sonnet 4 | Enable 1 million token context window | | Computer Use (Latest) | `computer-use-2025-01-24` | Claude 3.7 Sonnet | Latest computer use tools | | Computer Use (Legacy) | `computer-use-2024-10-22` | Claude 3.5 Sonnet v2 | Computer use tools for Claude 3.5 | | Token-Efficient Tools | `token-efficient-tools-2025-02-19` | Claude 3.7 Sonnet | More efficient tool usage | diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 66c8e55c3ae..186306b5b56 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23510,6 +23510,25 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, + "openrouter/anthropic/claude-opus-4.6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 3.75e-06, From 26d27803eb487a1b6b7881e2bdb74775b23c824b Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 19 Feb 2026 15:17:27 -0300 Subject: [PATCH 05/46] fix(models): disable function calling for PublicAI Apertus models The Apertus 8B and 70B models do not support standard OpenAI-style tool calling. Per Swiss AI's docs, tool use integration into inference engines is still in development. Set supports_function_calling and supports_tool_choice to false. Fixes #21124 --- model_prices_and_context_window.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 41acb5c8101..0ee2d586f85 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25907,8 +25907,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/swiss-ai/apertus-70b-instruct": { "input_cost_per_token": 0.0, @@ -25919,8 +25919,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/aisingapore/Gemma-SEA-LION-v4-27B-IT": { "input_cost_per_token": 0.0, From c6f60bed71f9198ed697c698bce533dbb9149106 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Mon, 23 Feb 2026 19:44:30 +0530 Subject: [PATCH 06/46] perf(spendlogs): optimize old spendlog deletion cron job --- .../litellm_proxy_extras/schema.prisma | 1 + .../db_transaction_queue/spend_log_cleanup.py | 30 ++++++------ litellm/proxy/schema.prisma | 1 + .../proxy/test_spend_log_cleanup.py | 47 ++++++------------- 4 files changed, 32 insertions(+), 47 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 4af7484148c..40bcaff67d6 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -500,6 +500,7 @@ model LiteLLM_SpendLogs { agent_id String? proxy_server_request Json? @default("{}") @@index([startTime]) + @@index([startTime, request_id]) @@index([end_user]) @@index([session_id]) } diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index 02fa84bae30..8c59c79ff0a 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -76,27 +76,29 @@ class SpendLogCleanup: "Max logs deleted - 1,00,000, rest of the logs will be deleted in next run" ) break - # Step 1: Find logs to delete - logs_to_delete = await prisma_client.db.litellm_spendlogs.find_many( - where={"startTime": {"lt": cutoff_date}}, - take=self.batch_size, + # Step 1: Find logs and delete them in one go without fetching to application + # Delete in batches, limited by self.batch_size + deleted_count = await prisma_client.db.execute_raw( + """ + DELETE FROM "LiteLLM_SpendLogs" + WHERE "request_id" IN ( + SELECT "request_id" FROM "LiteLLM_SpendLogs" + WHERE "startTime" < $1::timestamptz + LIMIT $2 + ) + """, + cutoff_date, + self.batch_size, ) - verbose_proxy_logger.info(f"Found {len(logs_to_delete)} logs in this batch") + verbose_proxy_logger.info(f"Deleted {deleted_count} logs in this batch") - if not logs_to_delete: + if deleted_count == 0: verbose_proxy_logger.info( f"No more logs to delete. Total deleted: {total_deleted}" ) break - request_ids = [log.request_id for log in logs_to_delete] - - # Step 2: Delete them in one go - await prisma_client.db.litellm_spendlogs.delete_many( - where={"request_id": {"in": request_ids}} - ) - - total_deleted += len(logs_to_delete) + total_deleted += deleted_count run_count += 1 # Add a small sleep to prevent overwhelming the database diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 50c0a55a875..7a1010d95c0 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -499,6 +499,7 @@ model LiteLLM_SpendLogs { agent_id String? proxy_server_request Json? @default("{}") @@index([startTime]) + @@index([startTime, request_id]) @@index([end_user]) @@index([session_id]) } diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py index 1ffbb83caef..c1fa3ad0c43 100644 --- a/tests/test_litellm/proxy/test_spend_log_cleanup.py +++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py @@ -151,28 +151,16 @@ async def test_should_delete_spend_logs(): @pytest.mark.asyncio async def test_cleanup_old_spend_logs_batch_deletion(): - from types import SimpleNamespace - from unittest.mock import AsyncMock, MagicMock, patch + from unittest.mock import AsyncMock, MagicMock # Setup Prisma client mock_prisma_client = MagicMock() mock_db = MagicMock() - # Mock spendlogs table - mock_spendlogs = MagicMock() - mock_spendlogs.find_many = AsyncMock() - mock_spendlogs.delete_many = AsyncMock() - - # Create 1500 mocked logs with .request_id - mock_logs = [SimpleNamespace(request_id=f"req_{i}") for i in range(1500)] - mock_spendlogs.find_many.side_effect = [ - mock_logs[:1000], # Batch 1 - mock_logs[1000:], # Batch 2 - [], # Done - ] + # Mock execute_raw to return deleted counts + mock_db.execute_raw = AsyncMock(side_effect=[1000, 500, 0]) # Wire up mocks - mock_db.litellm_spendlogs = mock_spendlogs mock_prisma_client.db = mock_db # Mock Redis cache and pod_lock_manager @@ -189,15 +177,13 @@ async def test_cleanup_old_spend_logs_batch_deletion(): assert cleaner._should_delete_spend_logs() is True await cleaner.cleanup_old_spend_logs(mock_prisma_client) - # Validate batching and deletion - assert mock_spendlogs.find_many.call_count == 3 - assert mock_spendlogs.delete_many.call_count == 2 - mock_spendlogs.delete_many.assert_any_call( - where={"request_id": {"in": [f"req_{i}" for i in range(1000)]}} - ) - mock_spendlogs.delete_many.assert_any_call( - where={"request_id": {"in": [f"req_{i}" for i in range(1000, 1500)]}} - ) + # Validate batching and deletion via raw SQL + assert mock_db.execute_raw.call_count == 3 + + # Check the first call argument + call_args_sql = mock_db.execute_raw.call_args_list[0][0][0] + assert 'DELETE FROM "LiteLLM_SpendLogs"' in call_args_sql + assert 'WHERE "request_id" IN' in call_args_sql @pytest.mark.asyncio @@ -208,10 +194,7 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff(): # Setup Prisma client mock_prisma_client = MagicMock() mock_db = MagicMock() - mock_spendlogs = MagicMock() - mock_spendlogs.find_many = AsyncMock(return_value=[]) - mock_spendlogs.delete_many = AsyncMock() - mock_db.litellm_spendlogs = mock_spendlogs + mock_db.execute_raw = AsyncMock(return_value=0) mock_prisma_client.db = mock_db # Mock Redis cache and pod_lock_manager @@ -229,7 +212,7 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff(): await cleaner.cleanup_old_spend_logs(mock_prisma_client) # Verify the cutoff date is correct - cutoff_date = mock_spendlogs.find_many.call_args[1]["where"]["startTime"]["lt"] + cutoff_date = mock_db.execute_raw.call_args[0][1] expected_cutoff = datetime.now(timezone.utc) - timedelta(seconds=86400) assert ( abs((cutoff_date - expected_cutoff).total_seconds()) < 1 @@ -242,14 +225,12 @@ async def test_cleanup_old_spend_logs_no_retention_period(): Test that no logs are deleted when no retention period is set """ mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_spendlogs.find_many = AsyncMock() - mock_prisma_client.db.litellm_spendlogs.delete = AsyncMock() + mock_prisma_client.db.execute_raw = AsyncMock() cleaner = SpendLogCleanup(general_settings={}) # no retention await cleaner.cleanup_old_spend_logs(mock_prisma_client) - mock_prisma_client.db.litellm_spendlogs.find_many.assert_not_called() - mock_prisma_client.db.litellm_spendlogs.delete.assert_not_called() + mock_prisma_client.db.execute_raw.assert_not_called() def test_cleanup_batch_size_env_var(monkeypatch): From f453427264866e038d9b714701fbce925e0fe6c6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 25 Feb 2026 17:19:33 +0530 Subject: [PATCH 07/46] Add v1 for anthropic responses transformation --- .../messages/handler.py | 59 ++- .../responses_adapters/__init__.py | 3 + .../responses_adapters/handler.py | 213 +++++++++ .../responses_adapters/streaming_iterator.py | 265 ++++++++++++ .../responses_adapters/transformation.py | 407 ++++++++++++++++++ ...erimental_pass_through_messages_handler.py | 62 +-- 6 files changed, 958 insertions(+), 51 deletions(-) create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 7e5a4f22a7f..6fe0fcd4fdf 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -25,8 +25,18 @@ from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler +from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler from .utils import AnthropicMessagesRequestUtils, mock_response +# Providers that are routed directly to the OpenAI Responses API instead of +# going through chat/completions. +_RESPONSES_API_PROVIDERS = frozenset({"openai", "azure", "azure_text"}) + + +def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool: + """Return True when the provider should use the Responses API path.""" + return custom_llm_provider in _RESPONSES_API_PROVIDERS + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -282,29 +292,34 @@ def anthropic_messages_handler( ) ) if anthropic_messages_provider_config is None: - # Handle non-Anthropic models using the adapter - return ( - LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( - max_tokens=max_tokens, - messages=messages, - model=model, - metadata=metadata, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - _is_async=is_async, - api_key=api_key, - api_base=api_base, - client=client, - custom_llm_provider=custom_llm_provider, - **kwargs, + # Route to Responses API for OpenAI / Azure, chat/completions for everything else. + _shared_kwargs = dict( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + _is_async=is_async, + api_key=api_key, + api_base=api_base, + client=client, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + if _should_route_to_responses_api(custom_llm_provider): + return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + **_shared_kwargs ) + return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( + **_shared_kwargs ) if custom_llm_provider is None: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py new file mode 100644 index 00000000000..6ad3c7b0164 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py @@ -0,0 +1,3 @@ +from .transformation import LiteLLMAnthropicToResponsesAPIAdapter + +__all__ = ["LiteLLMAnthropicToResponsesAPIAdapter"] diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py new file mode 100644 index 00000000000..18dbabb1e14 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -0,0 +1,213 @@ +""" +Handler for the Anthropic v1/messages -> OpenAI Responses API path. + +Used when the target model is an OpenAI or Azure model. +""" + +from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union + +import litellm +from litellm.types.llms.anthropic import AnthropicMessagesRequest +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, +) +from litellm.types.llms.openai import ResponsesAPIResponse + +from .streaming_iterator import AnthropicResponsesStreamWrapper +from .transformation import LiteLLMAnthropicToResponsesAPIAdapter + +_ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter() + + +def _build_responses_kwargs( + *, + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + extra_kwargs: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). + """ + # Build a typed AnthropicMessagesRequest for the adapter + request_data: Dict[str, Any] = {"model": model, "messages": messages, "max_tokens": max_tokens} + if metadata: + request_data["metadata"] = metadata + if system: + request_data["system"] = system + if temperature is not None: + request_data["temperature"] = temperature + if thinking: + request_data["thinking"] = thinking + if tool_choice: + request_data["tool_choice"] = tool_choice + if tools: + request_data["tools"] = tools + if top_p is not None: + request_data["top_p"] = top_p + if output_format: + request_data["output_format"] = output_format + + anthropic_request = AnthropicMessagesRequest(**request_data) + responses_kwargs = _ADAPTER.translate_request(anthropic_request) + + if stream: + responses_kwargs["stream"] = True + + # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) + excluded = {"anthropic_messages"} + for key, value in (extra_kwargs or {}).items(): + if key == "litellm_logging_obj" and value is not None: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObject, + ) + from litellm.types.utils import CallTypes + + if isinstance(value, LiteLLMLoggingObject): + # Reclassify as acompletion so the success handler doesn't try to + # validate the Responses API event as an AnthropicResponse. + # (Mirrors the pattern used in LiteLLMMessagesToCompletionTransformationHandler.) + setattr(value, "call_type", CallTypes.acompletion.value) + responses_kwargs[key] = value + elif key not in excluded and key not in responses_kwargs and value is not None: + responses_kwargs[key] = value + + return responses_kwargs + + +class LiteLLMMessagesToResponsesAPIHandler: + """ + Handles Anthropic /v1/messages requests for OpenAI / Azure models by + calling litellm.responses() / litellm.aresponses() directly and translating + the response back to Anthropic format. + """ + + @staticmethod + async def async_anthropic_messages_handler( + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + **kwargs, + ) -> Union[AnthropicMessagesResponse, AsyncIterator]: + responses_kwargs = _build_responses_kwargs( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + result = await litellm.aresponses(**responses_kwargs) + + if stream: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + return wrapper.async_anthropic_sse_wrapper() + + if not isinstance(result, ResponsesAPIResponse): + raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") + + return _ADAPTER.translate_response(result) + + @staticmethod + def anthropic_messages_handler( + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + _is_async: bool = False, + **kwargs, + ) -> Union[ + AnthropicMessagesResponse, + AsyncIterator[Any], + Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]], + ]: + if _is_async: + return LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + **kwargs, + ) + + # Sync path + responses_kwargs = _build_responses_kwargs( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + result = litellm.responses(**responses_kwargs) + + if stream: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + return wrapper.async_anthropic_sse_wrapper() + + if not isinstance(result, ResponsesAPIResponse): + raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") + + return _ADAPTER.translate_response(result) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py new file mode 100644 index 00000000000..0e6268e82f3 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -0,0 +1,265 @@ +# What is this? +## Translates OpenAI call to Anthropic `/v1/messages` format +import json +import traceback +from collections import deque +from typing import Any, AsyncIterator, Dict + +from litellm import verbose_logger +from litellm._uuid import uuid + + +class AnthropicResponsesStreamWrapper: + """ + Wraps a Responses API streaming iterator and re-emits events in Anthropic SSE format. + + Responses API event flow (relevant subset): + response.created -> message_start + response.output_item.added -> content_block_start (if message/function_call) + response.output_text.delta -> content_block_delta (text_delta) + response.reasoning_summary_text.delta -> content_block_delta (thinking_delta) + response.function_call_arguments.delta -> content_block_delta (input_json_delta) + response.output_item.done -> content_block_stop + response.completed -> message_delta + message_stop + """ + + def __init__( + self, + responses_stream: Any, + model: str, + ) -> None: + self.responses_stream = responses_stream + self.model = model + self._message_id: str = f"msg_{uuid.uuid4()}" + self._current_block_index: int = -1 + # Map item_id -> content_block_index so we can stop the right block later + self._item_id_to_block_index: Dict[str, int] = {} + # Track open function_call items by item_id so we can emit tool_use start + self._pending_tool_ids: Dict[str, str] = {} # item_id -> call_id / name accumulator + self._sent_message_start = False + self._sent_message_stop = False + self._chunk_queue: deque = deque() + + def _make_message_start(self) -> Dict[str, Any]: + return { + "type": "message_start", + "message": { + "id": self._message_id, + "type": "message", + "role": "assistant", + "content": [], + "model": self.model, + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, + } + + def _next_block_index(self) -> int: + self._current_block_index += 1 + return self._current_block_index + + def _process_event(self, event: Any) -> None: + """Convert one Responses API event into zero or more Anthropic chunks queued for emission.""" + event_type = getattr(event, "type", None) + if event_type is None and isinstance(event, dict): + event_type = event.get("type") + + if event_type is None: + return + + # ---- message_start ---- + if event_type == "response.created": + self._sent_message_start = True + self._chunk_queue.append(self._make_message_start()) + return + + # ---- content_block_start for a new output message item ---- + if event_type == "response.output_item.added": + item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None) + if item is None: + return + item_type = getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None) + item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) + + if item_type == "message": + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": {"type": "text", "text": ""}, + }) + elif item_type == "function_call": + call_id = getattr(item, "call_id", None) or (item.get("call_id") if isinstance(item, dict) else None) or "" + name = getattr(item, "name", None) or (item.get("name") if isinstance(item, dict) else None) or "" + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._pending_tool_ids[item_id] = call_id + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": { + "type": "tool_use", + "id": call_id, + "name": name, + "input": {}, + }, + }) + elif item_type == "reasoning": + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": {"type": "thinking", "thinking": ""}, + }) + return + + # ---- text delta ---- + if event_type == "response.output_text.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "text_delta", "text": delta}, + }) + return + + # ---- reasoning summary text delta ---- + if event_type == "response.reasoning_summary_text.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "thinking_delta", "thinking": delta}, + }) + return + + # ---- function call arguments delta ---- + if event_type == "response.function_call_arguments.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "input_json_delta", "partial_json": delta}, + }) + return + + # ---- output item done -> content_block_stop ---- + if event_type == "response.output_item.done": + item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None) + item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_stop", + "index": block_idx, + }) + return + + # ---- response completed -> message_delta + message_stop ---- + if event_type in ("response.completed", "response.failed", "response.incomplete"): + response_obj = getattr(event, "response", None) or (event.get("response") if isinstance(event, dict) else None) + stop_reason = "end_turn" + input_tokens = 0 + output_tokens = 0 + cache_creation_tokens = 0 + cache_read_tokens = 0 + + if response_obj is not None: + status = getattr(response_obj, "status", None) + if status == "incomplete": + stop_reason = "max_tokens" + usage = getattr(response_obj, "usage", None) + if usage is not None: + input_tokens = getattr(usage, "input_tokens", 0) or 0 + output_tokens = getattr(usage, "output_tokens", 0) or 0 + cache_creation_tokens = getattr(usage, "input_tokens_details", None) + cache_read_tokens = getattr(usage, "output_tokens_details", None) + # Prefer direct cache fields if present + cache_creation_tokens = getattr(usage, "cache_creation_input_tokens", 0) or 0 + cache_read_tokens = getattr(usage, "cache_read_input_tokens", 0) or 0 + + # Check if tool_use was in the output to override stop_reason + if response_obj is not None: + output = getattr(response_obj, "output", []) or [] + for out_item in output: + out_type = getattr(out_item, "type", None) or (out_item.get("type") if isinstance(out_item, dict) else None) + if out_type == "function_call": + stop_reason = "tool_use" + break + + usage_delta: Dict[str, Any] = { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } + if cache_creation_tokens: + usage_delta["cache_creation_input_tokens"] = cache_creation_tokens + if cache_read_tokens: + usage_delta["cache_read_input_tokens"] = cache_read_tokens + + self._chunk_queue.append({ + "type": "message_delta", + "delta": {"stop_reason": stop_reason, "stop_sequence": None}, + "usage": usage_delta, + }) + self._chunk_queue.append({"type": "message_stop"}) + self._sent_message_stop = True + return + + def __aiter__(self) -> "AnthropicResponsesStreamWrapper": + return self + + async def __anext__(self) -> Dict[str, Any]: + # Return any queued chunks first + if self._chunk_queue: + return self._chunk_queue.popleft() + + # Emit message_start if not yet done (fallback if response.created wasn't fired) + if not self._sent_message_start: + self._sent_message_start = True + self._chunk_queue.append(self._make_message_start()) + return self._chunk_queue.popleft() + + # Consume the upstream stream + try: + async for event in self.responses_stream: + self._process_event(event) + if self._chunk_queue: + return self._chunk_queue.popleft() + except StopAsyncIteration: + pass + except Exception as e: + verbose_logger.error( + f"AnthropicResponsesStreamWrapper error: {e}\n{traceback.format_exc()}" + ) + + # Drain any remaining queued chunks + if self._chunk_queue: + return self._chunk_queue.popleft() + + raise StopAsyncIteration + + async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]: + """Yield SSE-encoded bytes for each Anthropic event chunk.""" + async for chunk in self: + if isinstance(chunk, dict): + event_type: str = str(chunk.get("type", "message")) + payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n" + yield payload.encode() + else: + yield chunk diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py new file mode 100644 index 00000000000..a428e8f4e8f --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -0,0 +1,407 @@ +""" +Transformation layer: Anthropic /v1/messages <-> OpenAI Responses API. + +This module owns all format conversions for the direct v1/messages -> Responses API +path used for OpenAI and Azure models. +""" + +import json +from typing import Any, Dict, List, Optional, Union, cast + +from litellm.types.llms.anthropic import ( + AllAnthropicToolsValues, + AnthopicMessagesAssistantMessageParam, + AnthropicFinishReason, + AnthropicMessagesRequest, + AnthropicMessagesToolChoice, + AnthropicMessagesUserMessageParam, + AnthropicResponseContentBlockText, + AnthropicResponseContentBlockThinking, + AnthropicResponseContentBlockToolUse, +) +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, + AnthropicUsage, +) +from litellm.types.llms.openai import ResponsesAPIResponse + + +class LiteLLMAnthropicToResponsesAPIAdapter: + """ + Converts Anthropic /v1/messages requests to OpenAI Responses API format and + converts Responses API responses back to Anthropic format. + """ + + # ------------------------------------------------------------------ # + # Request translation: Anthropic -> Responses API # + # ------------------------------------------------------------------ # + + @staticmethod + def _translate_anthropic_image_source_to_url(source: dict) -> Optional[str]: + """Convert Anthropic image source to a URL string.""" + source_type = source.get("type") + if source_type == "base64": + media_type = source.get("media_type", "image/jpeg") + data = source.get("data", "") + return f"data:{media_type};base64,{data}" if data else None + elif source_type == "url": + return source.get("url") + return None + + def translate_messages_to_responses_input( + self, + messages: List[ + Union[ + AnthropicMessagesUserMessageParam, + AnthopicMessagesAssistantMessageParam, + ] + ], + ) -> List[Dict[str, Any]]: + """ + Convert Anthropic messages list to Responses API `input` items. + + Mapping: + user text -> message(role=user, input_text) + user image -> message(role=user, input_image) + user tool_result -> function_call_output + assistant text -> message(role=assistant, output_text) + assistant tool_use -> function_call + """ + input_items: List[Dict[str, Any]] = [] + + for m in messages: + role = m["role"] + content = m.get("content") + + if role == "user": + if isinstance(content, str): + input_items.append({ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": content}], + }) + elif isinstance(content, list): + user_parts: List[Dict[str, Any]] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "text": + user_parts.append({"type": "input_text", "text": block.get("text", "")}) + elif btype == "image": + url = self._translate_anthropic_image_source_to_url(block.get("source", {})) + if url: + user_parts.append({"type": "input_image", "image_url": url}) + elif btype == "tool_result": + tool_use_id = block.get("tool_use_id", "") + inner = block.get("content") + if inner is None: + output_text = "" + elif isinstance(inner, str): + output_text = inner + elif isinstance(inner, list): + parts = [ + c.get("text", "") + for c in inner + if isinstance(c, dict) and c.get("type") == "text" + ] + output_text = "\n".join(parts) + else: + output_text = str(inner) + # tool_result is a top-level item, not inside the message + input_items.append({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": output_text, + }) + if user_parts: + input_items.append({ + "type": "message", + "role": "user", + "content": user_parts, + }) + + elif role == "assistant": + if isinstance(content, str): + input_items.append({ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": content}], + }) + elif isinstance(content, list): + asst_parts: List[Dict[str, Any]] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "text": + asst_parts.append({"type": "output_text", "text": block.get("text", "")}) + elif btype == "tool_use": + # tool_use becomes a top-level function_call item + input_items.append({ + "type": "function_call", + "call_id": block.get("id", ""), + "name": block.get("name", ""), + "arguments": json.dumps(block.get("input", {})), + }) + elif btype == "thinking": + thinking_text = block.get("thinking", "") + if thinking_text: + asst_parts.append({"type": "output_text", "text": thinking_text}) + if asst_parts: + input_items.append({ + "type": "message", + "role": "assistant", + "content": asst_parts, + }) + + return input_items + + def translate_tools_to_responses_api( + self, + tools: List[AllAnthropicToolsValues], + ) -> List[Dict[str, Any]]: + """Convert Anthropic tool definitions to Responses API function tools.""" + result: List[Dict[str, Any]] = [] + for tool in tools: + tool_dict = cast(Dict[str, Any], tool) + tool_type = tool_dict.get("type", "") + tool_name = tool_dict.get("name", "") + # web_search tool + if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search": + result.append({"type": "web_search_preview"}) + continue + func_tool: Dict[str, Any] = {"type": "function", "name": tool_name} + if "description" in tool_dict: + func_tool["description"] = tool_dict["description"] + if "input_schema" in tool_dict: + func_tool["parameters"] = tool_dict["input_schema"] + result.append(func_tool) + return result + + @staticmethod + def translate_tool_choice_to_responses_api( + tool_choice: AnthropicMessagesToolChoice, + ) -> Dict[str, Any]: + """Convert Anthropic tool_choice to Responses API tool_choice.""" + tc_type = tool_choice.get("type") + if tc_type == "any": + return {"type": "required"} + elif tc_type == "tool": + return {"type": "function", "name": tool_choice.get("name", "")} + return {"type": "auto"} + + @staticmethod + def translate_thinking_to_reasoning(thinking: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """ + Convert Anthropic thinking param to Responses API reasoning param. + + thinking.budget_tokens maps to reasoning effort: + >= 10000 -> high, >= 5000 -> medium, >= 2000 -> low, < 2000 -> minimal + """ + if not isinstance(thinking, dict) or thinking.get("type") != "enabled": + return None + budget = thinking.get("budget_tokens", 0) + if budget >= 10000: + effort = "high" + elif budget >= 5000: + effort = "medium" + elif budget >= 2000: + effort = "low" + else: + effort = "minimal" + return {"effort": effort, "summary": "detailed"} + + def translate_request( + self, + anthropic_request: AnthropicMessagesRequest, + ) -> Dict[str, Any]: + """ + Translate a full Anthropic /v1/messages request dict to + litellm.responses() / litellm.aresponses() kwargs. + """ + model: str = anthropic_request["model"] + messages_list = cast( + List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]], + anthropic_request["messages"], + ) + + responses_kwargs: Dict[str, Any] = { + "model": model, + "input": self.translate_messages_to_responses_input(messages_list), + } + + # system -> instructions + system = anthropic_request.get("system") + if system: + if isinstance(system, str): + responses_kwargs["instructions"] = system + elif isinstance(system, list): + text_parts = [ + b.get("text", "") + for b in system + if isinstance(b, dict) and b.get("type") == "text" + ] + responses_kwargs["instructions"] = "\n".join(filter(None, text_parts)) + + # max_tokens -> max_output_tokens + max_tokens = anthropic_request.get("max_tokens") + if max_tokens: + responses_kwargs["max_output_tokens"] = max_tokens + + # temperature / top_p passed through + if "temperature" in anthropic_request: + responses_kwargs["temperature"] = anthropic_request["temperature"] + if "top_p" in anthropic_request: + responses_kwargs["top_p"] = anthropic_request["top_p"] + + # tools + tools = anthropic_request.get("tools") + if tools: + responses_kwargs["tools"] = self.translate_tools_to_responses_api( + cast(List[AllAnthropicToolsValues], tools) + ) + + # tool_choice + tool_choice = anthropic_request.get("tool_choice") + if tool_choice: + responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api( + cast(AnthropicMessagesToolChoice, tool_choice) + ) + + # thinking -> reasoning + thinking = anthropic_request.get("thinking") + if isinstance(thinking, dict): + reasoning = self.translate_thinking_to_reasoning(thinking) + if reasoning: + responses_kwargs["reasoning"] = reasoning + + # output_format -> text format + output_format = anthropic_request.get("output_format") + if isinstance(output_format, dict) and output_format.get("type") == "json_schema": + schema = output_format.get("schema") + if schema: + responses_kwargs["text"] = { + "format": { + "type": "json_schema", + "name": "structured_output", + "schema": schema, + "strict": True, + } + } + + # metadata user_id -> user + metadata = anthropic_request.get("metadata") + if isinstance(metadata, dict) and "user_id" in metadata: + responses_kwargs["user"] = metadata["user_id"] + + return responses_kwargs + + # ------------------------------------------------------------------ # + # Response translation: Responses API -> Anthropic # + # ------------------------------------------------------------------ # + + def translate_response( + self, + response: ResponsesAPIResponse, + ) -> AnthropicMessagesResponse: + """ + Translate an OpenAI ResponsesAPIResponse to AnthropicMessagesResponse. + """ + from openai.types.responses import ( + ResponseFunctionToolCall, + ResponseOutputMessage, + ResponseReasoningItem, + ) + + from litellm.types.llms.openai import ResponseAPIUsage + + content: List[Dict[str, Any]] = [] + stop_reason: AnthropicFinishReason = "end_turn" + + for item in response.output: + if isinstance(item, ResponseReasoningItem): + for summary in item.summary: + text = getattr(summary, "text", "") + if text: + content.append( + AnthropicResponseContentBlockThinking( + type="thinking", + thinking=text, + signature=None, + ).model_dump() + ) + + elif isinstance(item, ResponseOutputMessage): + for part in item.content: + if getattr(part, "type", None) == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=getattr(part, "text", "") + ).model_dump() + ) + + elif isinstance(item, ResponseFunctionToolCall): + try: + input_data = json.loads(item.arguments) if item.arguments else {} + except (json.JSONDecodeError, TypeError): + input_data = {} + content.append( + AnthropicResponseContentBlockToolUse( + type="tool_use", + id=item.call_id or item.id, + name=item.name, + input=input_data, + ).model_dump() + ) + stop_reason = "tool_use" + + elif isinstance(item, dict): + item_type = item.get("type") + if item_type == "message": + for part in item.get("content", []): + if isinstance(part, dict) and part.get("type") == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=part.get("text", "") + ).model_dump() + ) + elif item_type == "function_call": + try: + input_data = json.loads(item.get("arguments", "{}")) + except (json.JSONDecodeError, TypeError): + input_data = {} + content.append( + AnthropicResponseContentBlockToolUse( + type="tool_use", + id=item.get("call_id") or item.get("id", ""), + name=item.get("name", ""), + input=input_data, + ).model_dump() + ) + stop_reason = "tool_use" + + # status -> stop_reason override + if response.status == "incomplete": + stop_reason = "max_tokens" + + # usage + raw_usage: Optional[ResponseAPIUsage] = response.usage + input_tokens = int(getattr(raw_usage, "input_tokens", 0) or 0) + output_tokens = int(getattr(raw_usage, "output_tokens", 0) or 0) + + anthropic_usage = AnthropicUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + return AnthropicMessagesResponse( + id=response.id, + type="message", + role="assistant", + model=response.model or "unknown-model", + stop_sequence=None, + usage=anthropic_usage, # type: ignore + content=content, # type: ignore + stop_reason=stop_reason, + ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 77c74a7847e..c671d9b37b8 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -16,13 +16,14 @@ from litellm.types.utils import Delta, ModelResponse, StreamingChoices def test_anthropic_experimental_pass_through_messages_handler(): """ - Test that api key is passed to litellm.completion + Test that api key is passed to litellm.responses for OpenAI models. + OpenAI and Azure models are routed directly to the Responses API. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=100, @@ -32,19 +33,20 @@ def test_anthropic_experimental_pass_through_messages_handler(): ) except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" + mock_responses.assert_called_once() + assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_and_api_base_and_custom_values(): """ - Test that api key is passed to litellm.completion + Test that api key, api base, and extra kwargs are forwarded to litellm.responses for Azure models. + Azure models are routed directly to the Responses API. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=100, @@ -56,10 +58,10 @@ def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_an ) except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" - assert mock_completion.call_args.kwargs["api_base"] == "test-api-base" - assert mock_completion.call_args.kwargs["custom_key"] == "custom_value" + mock_responses.assert_called_once() + assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" + assert mock_responses.call_args.kwargs["api_base"] == "test-api-base" + assert mock_responses.call_args.kwargs["custom_key"] == "custom_value" def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): @@ -143,19 +145,19 @@ async def test_bedrock_converse_budget_tokens_preserved(): assert thinking_param.get("budget_tokens") == 1024, f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" -def test_openai_model_with_thinking_converts_to_reasoning_effort(): +def test_openai_model_with_thinking_converts_to_reasoning(): """ - Test that when using a non-Anthropic model (like OpenAI gpt-5.2) with thinking parameter, - the thinking is converted to reasoning_effort and NOT passed as thinking. - - This ensures we don't regress on issue #16052 where non-Anthropic models would fail - with UnsupportedParamsError when thinking was passed directly. + Test that when using an OpenAI model with thinking parameter, the thinking is + converted to a Responses API `reasoning` param (NOT passed as thinking). + + OpenAI models are routed directly to the Responses API, so we verify that + litellm.responses() is called with `reasoning` properly set. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=1024, @@ -170,20 +172,22 @@ def test_openai_model_with_thinking_converts_to_reasoning_effort(): except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - - call_kwargs = mock_completion.call_args.kwargs - - # Verify reasoning_effort is set (converted from thinking) - assert "reasoning_effort" in call_kwargs, "reasoning_effort should be passed to completion" + mock_responses.assert_called_once() - # reasoning_effort is transformed into a dict with effort and summary fields - expected_reasoning_effort = {"effort": "minimal", "summary": "detailed"} - assert call_kwargs["reasoning_effort"] == expected_reasoning_effort, \ - f"reasoning_effort should be {expected_reasoning_effort} for budget_tokens=1024, got {call_kwargs.get('reasoning_effort')}" + call_kwargs = mock_responses.call_args.kwargs - # Verify thinking is NOT passed (non-Claude model) - assert "thinking" not in call_kwargs, "thinking should NOT be passed for non-Claude models" + # Verify reasoning is set (converted from thinking) + assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses" + + # budget_tokens=1024 -> effort="minimal" (< 2000 threshold) + expected_reasoning = {"effort": "minimal", "summary": "detailed"} + assert call_kwargs["reasoning"] == expected_reasoning, ( + f"reasoning should be {expected_reasoning} for budget_tokens=1024, " + f"got {call_kwargs.get('reasoning')}" + ) + + # Verify thinking is NOT passed directly to the Responses API + assert "thinking" not in call_kwargs, "thinking should NOT be passed directly to litellm.responses" class TestThinkingParameterTransformation: From d751fdc900887979352a915c8e931c96fef0029d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 11:13:26 +0530 Subject: [PATCH 08/46] add ChatCompletionImageObject in OpenAIChatCompletionAssistantMessage --- litellm/types/llms/openai.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 15e8d1be930..24b18cc488a 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -699,7 +699,15 @@ class OpenAIChatCompletionAssistantMessage(TypedDict, total=False): role: Required[Literal["assistant"]] content: Optional[ Union[ - str, Iterable[Union[ChatCompletionTextObject, ChatCompletionThinkingBlock]] + str, + Iterable[ + Union[ + ChatCompletionTextObject, + ChatCompletionThinkingBlock, + ChatCompletionRedactedThinkingBlock, + ChatCompletionImageObject, + ] + ], ] ] name: Optional[str] From f1080a7e3048ab580acb362cbdb6cb1da460f565 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 11:29:34 +0530 Subject: [PATCH 09/46] Add 'image_url; to both if the intent is to support it in assistant messages --- litellm/types/llms/openai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 24b18cc488a..c0aae9bc2de 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -794,17 +794,19 @@ ValidUserMessageContentTypes = [ "file", ] # used for validating user messages. Prevent users from accidentally sending anthropic messages. -# Assistant message content types (text, thinking, redacted_thinking) +# Assistant message content types (text, thinking, redacted_thinking, image_url) ValidAssistantMessageContentTypesLiteral = Literal[ "text", "thinking", "redacted_thinking", + "image_url", ] ValidAssistantMessageContentTypes = [ "text", "thinking", "redacted_thinking", + "image_url", ] # Combined valid content types for chat completion messages From 7adaf49db7a0fb413704666fca92aaa6b2ad2887 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 18:05:20 +0530 Subject: [PATCH 10/46] Add tranlation of context_management --- .../responses_adapters/handler.py | 16 +++++++ .../responses_adapters/transformation.py | 47 ++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py index 18dbabb1e14..c268d6c5be8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -24,7 +24,9 @@ def _build_responses_kwargs( max_tokens: int, messages: List[Dict], model: str, + context_management: Optional[Dict] = None, metadata: Optional[Dict] = None, + output_config: Optional[Dict] = None, stop_sequences: Optional[List[str]] = None, stream: Optional[bool] = False, system: Optional[str] = None, @@ -42,6 +44,10 @@ def _build_responses_kwargs( """ # Build a typed AnthropicMessagesRequest for the adapter request_data: Dict[str, Any] = {"model": model, "messages": messages, "max_tokens": max_tokens} + if context_management: + request_data["context_management"] = context_management + if output_config: + request_data["output_config"] = output_config if metadata: request_data["metadata"] = metadata if system: @@ -98,7 +104,9 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: List[Dict], model: str, + context_management: Optional[Dict] = None, metadata: Optional[Dict] = None, + output_config: Optional[Dict] = None, stop_sequences: Optional[List[str]] = None, stream: Optional[bool] = False, system: Optional[str] = None, @@ -115,7 +123,9 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens=max_tokens, messages=messages, model=model, + context_management=context_management, metadata=metadata, + output_config=output_config, stop_sequences=stop_sequences, stream=stream, system=system, @@ -145,7 +155,9 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: List[Dict], model: str, + context_management: Optional[Dict] = None, metadata: Optional[Dict] = None, + output_config: Optional[Dict] = None, stop_sequences: Optional[List[str]] = None, stream: Optional[bool] = False, system: Optional[str] = None, @@ -168,7 +180,9 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens=max_tokens, messages=messages, model=model, + context_management=context_management, metadata=metadata, + output_config=output_config, stop_sequences=stop_sequences, stream=stream, system=system, @@ -187,7 +201,9 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens=max_tokens, messages=messages, model=model, + context_management=context_management, metadata=metadata, + output_config=output_config, stop_sequences=stop_sequences, stream=stream, system=system, diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index a428e8f4e8f..c2752272905 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -191,6 +191,37 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return {"type": "function", "name": tool_choice.get("name", "")} return {"type": "auto"} + @staticmethod + def translate_context_management_to_responses_api( + context_management: Dict[str, Any], + ) -> Optional[List[Dict[str, Any]]]: + """ + Convert Anthropic context_management dict to OpenAI Responses API array format. + + Anthropic format: {"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]} + OpenAI format: [{"type": "compaction", "compact_threshold": 150000}] + """ + if not isinstance(context_management, dict): + return None + + edits = context_management.get("edits", []) + if not isinstance(edits, list): + return None + + result: List[Dict[str, Any]] = [] + for edit in edits: + if not isinstance(edit, dict): + continue + edit_type = edit.get("type", "") + if edit_type == "compact_20260112": + entry: Dict[str, Any] = {"type": "compaction"} + trigger = edit.get("trigger") + if isinstance(trigger, dict) and trigger.get("value") is not None: + entry["compact_threshold"] = int(trigger["value"]) + result.append(entry) + + return result if result else None + @staticmethod def translate_thinking_to_reasoning(thinking: Dict[str, Any]) -> Optional[Dict[str, Any]]: """ @@ -276,8 +307,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if reasoning: responses_kwargs["reasoning"] = reasoning - # output_format -> text format + # output_format / output_config.format -> text format + # output_format: {"type": "json_schema", "schema": {...}} + # output_config: {"format": {"type": "json_schema", "schema": {...}}} output_format = anthropic_request.get("output_format") + output_config = anthropic_request.get("output_config") + if not isinstance(output_format, dict) and isinstance(output_config, dict): + output_format = output_config.get("format") if isinstance(output_format, dict) and output_format.get("type") == "json_schema": schema = output_format.get("schema") if schema: @@ -290,10 +326,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter: } } + # context_management: Anthropic dict -> OpenAI array + context_management = anthropic_request.get("context_management") + if isinstance(context_management, dict): + openai_cm = self.translate_context_management_to_responses_api(context_management) + if openai_cm is not None: + responses_kwargs["context_management"] = openai_cm + # metadata user_id -> user metadata = anthropic_request.get("metadata") if isinstance(metadata, dict) and "user_id" in metadata: - responses_kwargs["user"] = metadata["user_id"] + responses_kwargs["user"] = str(metadata["user_id"])[:64] return responses_kwargs From 6a68e3bba3ab5d5845139c6ae7073a5366bacef1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 18:14:06 +0530 Subject: [PATCH 11/46] Add tests for messages to responses transformation: --- .../responses_adapters/__init__.py | 0 .../test_responses_adapters_transformation.py | 987 ++++++++++++++++++ 2 files changed, 987 insertions(+) create mode 100644 tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py create mode 100644 tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py new file mode 100644 index 00000000000..252ba230ff7 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -0,0 +1,987 @@ +""" +Tests for LiteLLMAnthropicToResponsesAPIAdapter +(litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py) +""" + +import json +import os +import sys +from typing import Any, Dict, List +from unittest.mock import MagicMock + +sys.path.insert(0, os.path.abspath("../../../../../../..")) + +from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import ( + LiteLLMAnthropicToResponsesAPIAdapter, +) +from litellm.types.llms.anthropic import AnthropicMessagesRequest + + +def _make_request(**overrides) -> AnthropicMessagesRequest: + base: dict = { + "model": "openai.gpt-5.1-codex", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 1024, + } + base.update(overrides) + return AnthropicMessagesRequest(**base) + + +_ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter() + + +# --------------------------------------------------------------------------- +# context_management conversion +# --------------------------------------------------------------------------- + + +class TestContextManagementConversion: + """Anthropic dict -> OpenAI array conversion for context_management.""" + + def test_compact_edit_converted_to_array(self): + """compact_20260112 with trigger maps to OpenAI compaction entry.""" + cm = { + "edits": [ + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 150000}, + } + ] + } + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result == [{"type": "compaction", "compact_threshold": 150000}] + + def test_compact_edit_without_trigger(self): + """compact_20260112 without a trigger still maps to a compaction entry.""" + cm = {"edits": [{"type": "compact_20260112"}]} + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result == [{"type": "compaction"}] + + def test_unknown_edit_type_is_dropped(self): + """Anthropic-only edit types (e.g. clear_thinking) are silently dropped.""" + cm = {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]} + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result is None + + def test_mixed_edits_only_known_types_kept(self): + """Only compact_20260112 is converted; unknown types are dropped.""" + cm = { + "edits": [ + {"type": "clear_thinking_20251015", "keep": "all"}, + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 200000}, + }, + ] + } + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result == [{"type": "compaction", "compact_threshold": 200000}] + + def test_non_dict_returns_none(self): + result = _ADAPTER.translate_context_management_to_responses_api([]) # type: ignore + assert result is None + + def test_translate_request_includes_context_management(self): + """translate_request converts context_management and sets it on kwargs.""" + req = _make_request( + context_management={ + "edits": [ + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 100000}, + } + ] + } + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["context_management"] == [ + {"type": "compaction", "compact_threshold": 100000} + ] + + def test_translate_request_drops_anthropic_only_context_management(self): + """context_management with only unknown edit types is omitted from kwargs.""" + req = _make_request( + context_management={ + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + } + ) + kwargs = _ADAPTER.translate_request(req) + assert "context_management" not in kwargs + + +# --------------------------------------------------------------------------- +# structured output via output_config +# --------------------------------------------------------------------------- + + +class TestOutputConfigStructuredOutput: + """output_config.format.json_schema -> OpenAI text.format conversion.""" + + _SCHEMA = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string"}, + }, + "required": ["name", "email"], + "additionalProperties": False, + } + + def test_output_config_format_json_schema_converted(self): + """output_config.format.json_schema is converted to OpenAI text.format.""" + req = _make_request( + output_config={"format": {"type": "json_schema", "schema": self._SCHEMA}} + ) + kwargs = _ADAPTER.translate_request(req) + assert "text" in kwargs + fmt = kwargs["text"]["format"] + assert fmt["type"] == "json_schema" + assert fmt["schema"] == self._SCHEMA + assert fmt["strict"] is True + assert fmt["name"] == "structured_output" + + def test_output_config_without_format_does_not_set_text(self): + """output_config with only non-format keys doesn't produce text.format.""" + req = _make_request(output_config={"effort": "high"}) + kwargs = _ADAPTER.translate_request(req) + assert "text" not in kwargs + + def test_output_format_still_works(self): + """The original output_format field still takes precedence when present.""" + req = _make_request( + output_format={"type": "json_schema", "schema": self._SCHEMA} + ) + kwargs = _ADAPTER.translate_request(req) + assert "text" in kwargs + assert kwargs["text"]["format"]["type"] == "json_schema" + + def test_output_format_takes_precedence_over_output_config(self): + """output_format takes precedence over output_config.format.""" + other_schema = {"type": "object", "properties": {"id": {"type": "integer"}}} + req = _make_request( + output_format={"type": "json_schema", "schema": self._SCHEMA}, + output_config={"format": {"type": "json_schema", "schema": other_schema}}, + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["text"]["format"]["schema"] == self._SCHEMA + + +# --------------------------------------------------------------------------- +# translate_messages_to_responses_input +# --------------------------------------------------------------------------- + +# Helper: cast plain dicts to the expected type so call sites stay clean. +def _translate_messages(messages: List[Any]) -> List[Dict[str, Any]]: + return _ADAPTER.translate_messages_to_responses_input(messages) # type: ignore[arg-type] + + +class TestTranslateMessagesToResponsesInput: + """Anthropic messages list -> OpenAI Responses API input items.""" + + def test_user_string_content(self): + """Plain string user message becomes a message with input_text.""" + messages = [{"role": "user", "content": "Hello world"}] + result = _translate_messages(messages) + assert result == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Hello world"}], + } + ] + + def test_user_list_text_block(self): + """User message with text content block maps to input_text.""" + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "What is 2+2?"}], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "What is 2+2?"}], + } + ] + + def test_user_multiple_text_blocks(self): + """Multiple text blocks in a user message are all converted.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "First part."}, + {"type": "text", "text": "Second part."}, + ], + } + ] + result = _translate_messages(messages) + assert len(result) == 1 + assert result[0]["content"] == [ + {"type": "input_text", "text": "First part."}, + {"type": "input_text", "text": "Second part."}, + ] + + def test_user_base64_image(self): + """User message with base64 image source becomes input_image with data URL.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc123", + }, + } + ], + } + ] + result = _translate_messages(messages) + assert len(result) == 1 + assert result[0]["content"] == [ + {"type": "input_image", "image_url": "data:image/png;base64,abc123"} + ] + + def test_user_url_image(self): + """User message with URL image source becomes input_image with the URL.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/img.jpg"}, + } + ], + } + ] + result = _translate_messages(messages) + assert result[0]["content"] == [ + {"type": "input_image", "image_url": "https://example.com/img.jpg"} + ] + + def test_user_base64_image_empty_data_skipped(self): + """Base64 image with empty data is skipped (no URL can be formed).""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/jpeg", "data": ""}, + } + ], + } + ] + result = _translate_messages(messages) + # No user_parts -> no message item appended + assert result == [] + + def test_user_tool_result_string_content(self): + """tool_result with string content becomes function_call_output.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_abc", + "content": "42 degrees", + } + ], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "function_call_output", + "call_id": "call_abc", + "output": "42 degrees", + } + ] + + def test_user_tool_result_list_content(self): + """tool_result with list of text blocks is joined into a single string.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_xyz", + "content": [ + {"type": "text", "text": "Line 1"}, + {"type": "text", "text": "Line 2"}, + ], + } + ], + } + ] + result = _translate_messages(messages) + assert result[0]["output"] == "Line 1\nLine 2" + + def test_user_tool_result_null_content(self): + """tool_result with null content becomes empty string output.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "call_null", "content": None} + ], + } + ] + result = _translate_messages(messages) + assert result[0]["output"] == "" + + def test_assistant_string_content(self): + """Plain string assistant message becomes a message with output_text.""" + messages = [{"role": "assistant", "content": "I can help with that."}] + result = _translate_messages(messages) + assert result == [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "I can help with that."}], + } + ] + + def test_assistant_text_block(self): + """Assistant message with text block maps to output_text.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "text", "text": "Here is the answer."}], + } + ] + result = _translate_messages(messages) + assert result[0]["content"] == [ + {"type": "output_text", "text": "Here is the answer."} + ] + + def test_assistant_tool_use_becomes_function_call(self): + """Assistant tool_use block becomes a top-level function_call item.""" + messages = [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01", + "name": "get_weather", + "input": {"location": "Boston"}, + } + ], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "function_call", + "call_id": "toolu_01", + "name": "get_weather", + "arguments": json.dumps({"location": "Boston"}), + } + ] + + def test_assistant_thinking_block_becomes_output_text(self): + """Assistant thinking block text is included as output_text.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Let me reason step by step."} + ], + } + ] + result = _translate_messages(messages) + assert result[0]["content"] == [ + {"type": "output_text", "text": "Let me reason step by step."} + ] + + def test_assistant_empty_thinking_block_skipped(self): + """Assistant thinking block with empty thinking text is skipped.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "thinking", "thinking": ""}], + } + ] + result = _translate_messages(messages) + assert result == [] + + def test_mixed_messages_ordering(self): + """Full multi-turn conversation is converted in order.""" + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_02", + "name": "get_weather", + "input": {"city": "NYC"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_02", + "content": "Sunny, 72F", + } + ], + }, + {"role": "assistant", "content": "It's sunny and 72°F in NYC."}, + ] + result = _translate_messages(messages) + types = [item["type"] for item in result] + assert types == ["message", "function_call", "function_call_output", "message"] + + def test_user_text_and_image_mixed(self): + """User message with both text and image produces both parts.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image:"}, + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/cat.jpg"}, + }, + ], + } + ] + result = _translate_messages(messages) + assert len(result) == 1 + assert result[0]["content"][0] == {"type": "input_text", "text": "Describe this image:"} + assert result[0]["content"][1] == { + "type": "input_image", + "image_url": "https://example.com/cat.jpg", + } + + def test_unknown_image_source_type_skipped(self): + """Image block with unknown source type is silently skipped.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"type": "file_path", "path": "/tmp/img.png"}, + } + ], + } + ] + result = _translate_messages(messages) + assert result == [] + + +# --------------------------------------------------------------------------- +# translate_tools_to_responses_api +# --------------------------------------------------------------------------- + + +class TestTranslateToolsToResponsesAPI: + """Anthropic tool definitions -> Responses API function tools.""" + + def test_regular_tool_with_description_and_schema(self): + """Standard tool with description and input_schema is converted to function.""" + tools = [ + { + "name": "get_weather", + "description": "Get current weather for a city.", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result == [ + { + "type": "function", + "name": "get_weather", + "description": "Get current weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ] + + def test_tool_without_description(self): + """Tool without a description omits the description key.""" + tools = [{"name": "ping", "input_schema": {"type": "object", "properties": {}}}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result[0]["type"] == "function" + assert result[0]["name"] == "ping" + assert "description" not in result[0] + + def test_tool_without_input_schema(self): + """Tool without input_schema omits the parameters key.""" + tools = [{"name": "no_schema_tool", "description": "Does something."}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result[0]["type"] == "function" + assert "parameters" not in result[0] + + def test_web_search_tool_by_name(self): + """Tool named 'web_search' maps to web_search_preview.""" + tools = [{"name": "web_search", "type": "custom"}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result == [{"type": "web_search_preview"}] + + def test_web_search_tool_by_type_prefix(self): + """Tool with type starting with 'web_search' maps to web_search_preview.""" + tools = [{"name": "search", "type": "web_search_20250305"}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result == [{"type": "web_search_preview"}] + + def test_multiple_tools_order_preserved(self): + """Multiple tools are converted in order.""" + tools = [ + {"name": "tool_a", "description": "A"}, + {"name": "web_search", "type": "custom"}, + {"name": "tool_b", "description": "B"}, + ] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert len(result) == 3 + assert result[0]["name"] == "tool_a" + assert result[1] == {"type": "web_search_preview"} + assert result[2]["name"] == "tool_b" + + def test_empty_tools_list(self): + """Empty tools list returns empty list.""" + assert _ADAPTER.translate_tools_to_responses_api([]) == [] + + +# --------------------------------------------------------------------------- +# translate_tool_choice_to_responses_api +# --------------------------------------------------------------------------- + + +class TestTranslateToolChoiceToResponsesAPI: + """Anthropic tool_choice -> Responses API tool_choice.""" + + def test_auto_maps_to_auto(self): + assert _ADAPTER.translate_tool_choice_to_responses_api({"type": "auto"}) == { + "type": "auto" + } + + def test_any_maps_to_required(self): + assert _ADAPTER.translate_tool_choice_to_responses_api({"type": "any"}) == { + "type": "required" + } + + def test_specific_tool_maps_to_function(self): + result = _ADAPTER.translate_tool_choice_to_responses_api( + {"type": "tool", "name": "get_weather"} + ) + assert result == {"type": "function", "name": "get_weather"} + + def test_unknown_type_defaults_to_auto(self): + result = _ADAPTER.translate_tool_choice_to_responses_api({"type": "none"}) + assert result == {"type": "auto"} + + +# --------------------------------------------------------------------------- +# translate_thinking_to_reasoning +# --------------------------------------------------------------------------- + + +class TestTranslateThinkingToReasoning: + """Anthropic thinking param -> Responses API reasoning param.""" + + def test_budget_high_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 10000} + ) + assert result == {"effort": "high", "summary": "detailed"} + + def test_budget_above_threshold_high_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 50000} + ) + assert result is not None + assert result["effort"] == "high" + + def test_budget_medium_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 7500} + ) + assert result == {"effort": "medium", "summary": "detailed"} + + def test_budget_low_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 3000} + ) + assert result == {"effort": "low", "summary": "detailed"} + + def test_budget_minimal_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 500} + ) + assert result == {"effort": "minimal", "summary": "detailed"} + + def test_budget_at_exact_thresholds(self): + result_medium = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 5000} + ) + assert result_medium is not None + assert result_medium["effort"] == "medium" + result_low = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 2000} + ) + assert result_low is not None + assert result_low["effort"] == "low" + + def test_disabled_type_returns_none(self): + result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}) + assert result is None + + def test_non_dict_returns_none(self): + result = _ADAPTER.translate_thinking_to_reasoning("enabled") # type: ignore + assert result is None + + def test_missing_budget_defaults_to_minimal(self): + """Missing budget_tokens defaults to 0, which is < 2000 -> minimal.""" + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}) + assert result == {"effort": "minimal", "summary": "detailed"} + + +# --------------------------------------------------------------------------- +# translate_request – broader coverage +# --------------------------------------------------------------------------- + + +class TestTranslateRequestBroaderCoverage: + """Full translate_request call: field-by-field mapping verification.""" + + def test_model_and_input_always_present(self): + req = _make_request() + kwargs = _ADAPTER.translate_request(req) + assert "model" in kwargs + assert "input" in kwargs + + def test_system_string_becomes_instructions(self): + req = _make_request(system="You are a helpful assistant.") + kwargs = _ADAPTER.translate_request(req) + assert kwargs["instructions"] == "You are a helpful assistant." + + def test_system_list_of_text_blocks_joined(self): + req = _make_request( + system=[ + {"type": "text", "text": "Be concise."}, + {"type": "text", "text": "Be helpful."}, + ] + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["instructions"] == "Be concise.\nBe helpful." + + def test_system_list_skips_non_text_blocks(self): + req = _make_request( + system=[ + {"type": "image", "source": {}}, + {"type": "text", "text": "Only text matters."}, + ] + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["instructions"] == "Only text matters." + + def test_max_tokens_mapped_to_max_output_tokens(self): + req = _make_request(max_tokens=512) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["max_output_tokens"] == 512 + + def test_temperature_passed_through(self): + req = _make_request(temperature=0.7) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["temperature"] == 0.7 + + def test_top_p_passed_through(self): + req = _make_request(top_p=0.9) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["top_p"] == 0.9 + + def test_tools_translated(self): + req = _make_request( + tools=[{"name": "calculator", "description": "Does math.", "input_schema": {}}] + ) + kwargs = _ADAPTER.translate_request(req) + assert len(kwargs["tools"]) == 1 + assert kwargs["tools"][0]["name"] == "calculator" + + def test_tool_choice_translated(self): + req = _make_request( + tools=[{"name": "do_thing"}], + tool_choice={"type": "tool", "name": "do_thing"}, + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["tool_choice"] == {"type": "function", "name": "do_thing"} + + def test_thinking_translated_to_reasoning(self): + req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["reasoning"] == {"effort": "high", "summary": "detailed"} + + def test_disabled_thinking_not_included_in_kwargs(self): + req = _make_request(thinking={"type": "disabled"}) + kwargs = _ADAPTER.translate_request(req) + assert "reasoning" not in kwargs + + def test_metadata_user_id_mapped_to_user(self): + req = _make_request(metadata={"user_id": "user-42"}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["user"] == "user-42" + + def test_metadata_user_id_truncated_to_64_chars(self): + long_id = "x" * 100 + req = _make_request(metadata={"user_id": long_id}) + kwargs = _ADAPTER.translate_request(req) + assert len(kwargs["user"]) == 64 + + def test_no_optional_fields_does_not_add_spurious_keys(self): + req = _make_request() + kwargs = _ADAPTER.translate_request(req) + for key in ("instructions", "temperature", "top_p", "tools", "tool_choice", + "reasoning", "text", "context_management", "user"): + assert key not in kwargs, f"unexpected key: {key}" + + +# --------------------------------------------------------------------------- +# translate_response +# --------------------------------------------------------------------------- + + +def _make_mock_response( + output: list, + status: str = "completed", + response_id: str = "resp_001", + model: str = "gpt-4o", + input_tokens: int = 100, + output_tokens: int = 50, +) -> MagicMock: + """Build a minimal mock ResponsesAPIResponse.""" + usage = MagicMock() + usage.input_tokens = input_tokens + usage.output_tokens = output_tokens + + resp = MagicMock() + resp.id = response_id + resp.model = model + resp.status = status + resp.output = output + resp.usage = usage + return resp + + +def _make_output_message(texts: List[str]) -> MagicMock: + """Build a mock ResponseOutputMessage with output_text parts.""" + from openai.types.responses import ResponseOutputMessage # type: ignore[import] + + parts = [] + for t in texts: + part = MagicMock() + part.type = "output_text" + part.text = t + parts.append(part) + + msg = MagicMock(spec=ResponseOutputMessage) + msg.content = parts + return msg + + +def _make_function_call_item( + call_id: str, name: str, arguments: str +) -> MagicMock: + """Build a mock ResponseFunctionToolCall.""" + from openai.types.responses import ResponseFunctionToolCall # type: ignore[import] + + item = MagicMock(spec=ResponseFunctionToolCall) + item.call_id = call_id + item.id = call_id + item.name = name + item.arguments = arguments + return item + + +def _make_reasoning_item(summaries: List[str]) -> MagicMock: + """Build a mock ResponseReasoningItem.""" + from openai.types.responses import ResponseReasoningItem # type: ignore[import] + + summary_mocks = [] + for text in summaries: + s = MagicMock() + s.text = text + summary_mocks.append(s) + + item = MagicMock(spec=ResponseReasoningItem) + item.summary = summary_mocks + return item + + +class TestTranslateResponse: + """Responses API -> AnthropicMessagesResponse conversion.""" + + def test_output_text_message_becomes_text_block(self): + """ResponseOutputMessage with output_text parts -> Anthropic text content.""" + response = _make_mock_response(output=[_make_output_message(["Hello!"])]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Hello!" + + def test_multiple_text_parts(self): + """Multiple output_text parts become multiple text content blocks.""" + response = _make_mock_response( + output=[_make_output_message(["Part 1", "Part 2"])] + ) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 2 + assert result["content"][0]["text"] == "Part 1" + assert result["content"][1]["text"] == "Part 2" + + def test_function_call_becomes_tool_use(self): + """ResponseFunctionToolCall -> Anthropic tool_use content block.""" + fc = _make_function_call_item("call_99", "get_weather", '{"city": "NYC"}') + response = _make_mock_response(output=[fc]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + block = result["content"][0] + assert block["type"] == "tool_use" + assert block["id"] == "call_99" + assert block["name"] == "get_weather" + assert block["input"] == {"city": "NYC"} + + def test_function_call_sets_stop_reason_tool_use(self): + """Presence of a function_call sets stop_reason to 'tool_use'.""" + fc = _make_function_call_item("call_1", "tool_a", "{}") + response = _make_mock_response(output=[fc]) + result: Any = _ADAPTER.translate_response(response) + assert result["stop_reason"] == "tool_use" + + def test_text_only_stop_reason_end_turn(self): + """Text-only response has stop_reason 'end_turn'.""" + response = _make_mock_response(output=[_make_output_message(["Hi"])]) + result: Any = _ADAPTER.translate_response(response) + assert result["stop_reason"] == "end_turn" + + def test_incomplete_status_sets_max_tokens(self): + """status='incomplete' overrides stop_reason to 'max_tokens'.""" + response = _make_mock_response( + output=[_make_output_message(["Truncated..."])], + status="incomplete", + ) + result: Any = _ADAPTER.translate_response(response) + assert result["stop_reason"] == "max_tokens" + + def test_reasoning_item_becomes_thinking_block(self): + """ResponseReasoningItem summaries -> Anthropic thinking content blocks.""" + reasoning = _make_reasoning_item(["Step 1: analyze. Step 2: conclude."]) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "thinking" + assert "Step 1" in result["content"][0]["thinking"] + + def test_empty_reasoning_summary_skipped(self): + """Reasoning item with empty text summary is not added to content.""" + reasoning = _make_reasoning_item([""]) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [] + + def test_usage_mapped_correctly(self): + """Input/output tokens from ResponseAPIUsage are mapped to AnthropicUsage.""" + response = _make_mock_response( + output=[_make_output_message(["OK"])], + input_tokens=200, + output_tokens=75, + ) + result: Any = _ADAPTER.translate_response(response) + assert result["usage"]["input_tokens"] == 200 + assert result["usage"]["output_tokens"] == 75 + + def test_model_and_id_preserved(self): + """Model and response ID from the Responses API are forwarded.""" + response = _make_mock_response( + output=[_make_output_message(["Hi"])], + response_id="resp_xyz", + model="gpt-4-turbo", + ) + result: Any = _ADAPTER.translate_response(response) + assert result["id"] == "resp_xyz" + assert result["model"] == "gpt-4-turbo" + + def test_role_is_always_assistant(self): + response = _make_mock_response(output=[_make_output_message(["Hi"])]) + result: Any = _ADAPTER.translate_response(response) + assert result["role"] == "assistant" + + def test_type_is_always_message(self): + response = _make_mock_response(output=[_make_output_message(["Hi"])]) + result: Any = _ADAPTER.translate_response(response) + assert result["type"] == "message" + + def test_empty_output_list(self): + """Empty output list produces empty content with 'end_turn' stop reason.""" + response = _make_mock_response(output=[]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [] + assert result["stop_reason"] == "end_turn" + + def test_function_call_with_invalid_json_arguments(self): + """Invalid JSON in function_call arguments falls back to empty dict.""" + fc = _make_function_call_item("call_bad", "broken_tool", "not-valid-json") + response = _make_mock_response(output=[fc]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["input"] == {} + + def test_dict_output_message_item(self): + """Dict-shaped output message (type=message) is also handled.""" + output_item = { + "type": "message", + "content": [{"type": "output_text", "text": "Dict-based response"}], + } + response = _make_mock_response(output=[output_item]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Dict-based response" + + def test_dict_function_call_item(self): + """Dict-shaped function_call item is converted to tool_use block.""" + output_item = { + "type": "function_call", + "call_id": "call_dict_1", + "name": "search", + "arguments": '{"query": "cats"}', + } + response = _make_mock_response(output=[output_item]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["type"] == "tool_use" + assert result["content"][0]["name"] == "search" + assert result["content"][0]["input"] == {"query": "cats"} + assert result["stop_reason"] == "tool_use" + + def test_mixed_reasoning_text_and_tool_use(self): + """Reasoning + text + tool_use in one response all convert correctly.""" + reasoning = _make_reasoning_item(["Thinking..."]) + text_msg = _make_output_message(["Here is my answer."]) + fc = _make_function_call_item("call_mix", "lookup", '{"id": 1}') + response = _make_mock_response(output=[reasoning, text_msg, fc]) + result: Any = _ADAPTER.translate_response(response) + types = [b["type"] for b in result["content"]] + assert "thinking" in types + assert "text" in types + assert "tool_use" in types + assert result["stop_reason"] == "tool_use" From f24a41898bba81bdb047e84fd66c1c9bc0f09832 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Thu, 26 Feb 2026 14:41:59 -0600 Subject: [PATCH 12/46] feat(vertex): add gemini-3.1-flash-image-preview model DB support - add gemini-3.1-flash-image-preview + vertex_ai alias entries\n- set pricing to Gemini 3.1 Flash Image Preview rates\n- mirror updates in packaged backup model map\n- update llm cost calc regression test to cover new model --- ...odel_prices_and_context_window_backup.json | 45 +++++++++++++++++++ model_prices_and_context_window.json | 45 +++++++++++++++++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 27 ++++++----- 3 files changed, 106 insertions(+), 11 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 57563fc0bcc..b21f23ac022 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14194,6 +14194,38 @@ "supports_vision": true, "supports_web_search": true }, + "gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true + }, "deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -31545,6 +31577,19 @@ "output_cost_per_token_batches": 6e-06, "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, + "vertex_ai/gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" + }, "vertex_ai/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 57563fc0bcc..b21f23ac022 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14194,6 +14194,38 @@ "supports_vision": true, "supports_web_search": true }, + "gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true + }, "deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -31545,6 +31577,19 @@ "output_cost_per_token_batches": 6e-06, "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, + "vertex_ai/gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" + }, "vertex_ai/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index b45cbbd99c0..9abce33fcd0 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -766,7 +766,14 @@ def test_service_tier_fallback_pricing(): assert abs(std_cost[1] - expected_standard_completion) < 1e-10, f"Standard completion cost mismatch: {std_cost[1]} vs {expected_standard_completion}" -def test_gemini_image_generation_cost_with_zero_text_tokens(): +@pytest.mark.parametrize( + "model", + [ + "gemini-3-pro-image-preview", + "gemini-3.1-flash-image-preview", + ], +) +def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): """ Test that image_tokens are correctly costed when text_tokens=0. @@ -779,7 +786,6 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini-3-pro-image-preview" custom_llm_provider = "vertex_ai" # Usage from the issue: text_tokens=0, image_tokens=1120, reasoning_tokens=225 @@ -809,9 +815,9 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(): # Expected costs: # - text_tokens: 0 * output_cost_per_token = 0 - # - image_tokens: 1120 * output_cost_per_image_token = 1120 * 1.2e-04 = 0.1344 - # - reasoning_tokens: 225 * output_cost_per_token = 225 * 1.2e-05 = 0.0027 - # Total completion: ~0.1371 + # - image_tokens: 1120 * output_cost_per_image_token + # - reasoning_tokens: 225 * output_cost_per_token + # Total completion should include both image + reasoning costs. output_cost_per_image_token = model_cost_map.get("output_cost_per_image_token", 0) output_cost_per_token = model_cost_map.get("output_cost_per_token", 0) @@ -820,12 +826,11 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(): expected_reasoning_cost = 225 * output_cost_per_token # reasoning uses base token cost expected_completion_cost = expected_image_cost + expected_reasoning_cost - # The bug was: all 1345 tokens were treated as text = 1345 * 1.2e-05 = 0.01614 - # Fixed: image_tokens use image pricing = ~0.137 - - assert completion_cost > 0.10, ( - f"Completion cost should be > $0.10 (image tokens are expensive), got ${completion_cost:.6f}. " - f"Bug: tokens may be incorrectly treated as text tokens." + # The bug was: all completion tokens were treated as text tokens only. + bugged_text_only_cost = 1345 * output_cost_per_token + assert completion_cost > bugged_text_only_cost * 2, ( + f"Completion cost should be significantly larger than text-only bugged path. " + f"Expected > {bugged_text_only_cost * 2:.6f}, got {completion_cost:.6f}" ) assert round(completion_cost, 4) == round(expected_completion_cost, 4), ( f"Expected completion cost ${expected_completion_cost:.6f}, got ${completion_cost:.6f}" From 702d5e88b8616fd1d13082be725612c482073568 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Thu, 26 Feb 2026 14:57:11 -0600 Subject: [PATCH 13/46] fix(cost): use token usage for gemini/vertex image generation when available - compute image_generation cost from usage token metadata for vertex/gemini\n- map ImageUsage to Usage and reuse generic_cost_per_token\n- fallback to output_cost_per_image when usage metadata missing\n- add tests for token-based path and fallback path --- .../image_generation/cost_calculator.py | 72 ++++++++- .../image_generation/cost_calculator.py | 71 ++++++++- .../llm_cost_calc/test_llm_cost_calc_utils.py | 144 ++++++++++++++++++ 3 files changed, 284 insertions(+), 3 deletions(-) diff --git a/litellm/llms/gemini/image_generation/cost_calculator.py b/litellm/llms/gemini/image_generation/cost_calculator.py index 0a9ca2e5276..6d7572d5522 100644 --- a/litellm/llms/gemini/image_generation/cost_calculator.py +++ b/litellm/llms/gemini/image_generation/cost_calculator.py @@ -2,10 +2,71 @@ Google AI Image Generation Cost Calculator """ -from typing import Any +from typing import Any, Optional import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.types.utils import ( + CompletionTokensDetailsWrapper, + ImageResponse, + PromptTokensDetailsWrapper, + Usage, +) + + +def _calculate_token_based_cost(model: str, image_response: ImageResponse) -> Optional[float]: + """ + Calculate token-based image generation cost when usage metadata is available. + + Falls back to None when usage metadata is missing/incomplete. + """ + usage = image_response.usage + if usage is None: + return None + + prompt_tokens = usage.input_tokens + completion_tokens = usage.output_tokens + total_tokens = usage.total_tokens + + if ( + prompt_tokens is None + or completion_tokens is None + or total_tokens is None + ): + return None + # ImageResponse may carry a default zeroed usage object even when provider + # usage metadata is absent. Treat this as missing usage and fall back. + if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: + return None + + input_tokens_details = getattr(usage, "input_tokens_details", None) + prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None + if input_tokens_details is not None: + prompt_tokens_details = PromptTokensDetailsWrapper( + text_tokens=getattr(input_tokens_details, "text_tokens", None), + image_tokens=getattr(input_tokens_details, "image_tokens", None), + cached_tokens=0, + ) + + normalized_usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + prompt_tokens_details=prompt_tokens_details, + completion_tokens_details=CompletionTokensDetailsWrapper( + text_tokens=0, + image_tokens=completion_tokens, + reasoning_tokens=0, + audio_tokens=0, + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=normalized_usage, + custom_llm_provider="gemini", + ) + return prompt_cost + completion_cost def cost_calculator( @@ -20,6 +81,13 @@ def cost_calculator( custom_llm_provider="gemini", ) + if isinstance(image_response, ImageResponse): + token_based_cost = _calculate_token_based_cost( + model=model, image_response=image_response + ) + if token_based_cost is not None: + return token_based_cost + output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if isinstance(image_response, ImageResponse): diff --git a/litellm/llms/vertex_ai/image_generation/cost_calculator.py b/litellm/llms/vertex_ai/image_generation/cost_calculator.py index 646c6080a2e..ac587182f04 100644 --- a/litellm/llms/vertex_ai/image_generation/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_generation/cost_calculator.py @@ -2,8 +2,71 @@ Vertex AI Image Generation Cost Calculator """ +from typing import Optional + import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.types.utils import ( + CompletionTokensDetailsWrapper, + ImageResponse, + PromptTokensDetailsWrapper, + Usage, +) + + +def _calculate_token_based_cost(model: str, image_response: ImageResponse) -> Optional[float]: + """ + Calculate token-based image generation cost when usage metadata is available. + + Falls back to None when usage metadata is missing/incomplete. + """ + usage = image_response.usage + if usage is None: + return None + + prompt_tokens = usage.input_tokens + completion_tokens = usage.output_tokens + total_tokens = usage.total_tokens + + if ( + prompt_tokens is None + or completion_tokens is None + or total_tokens is None + ): + return None + # ImageResponse may carry a default zeroed usage object even when provider + # usage metadata is absent. Treat this as missing usage and fall back. + if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: + return None + + input_tokens_details = getattr(usage, "input_tokens_details", None) + prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None + if input_tokens_details is not None: + prompt_tokens_details = PromptTokensDetailsWrapper( + text_tokens=getattr(input_tokens_details, "text_tokens", None), + image_tokens=getattr(input_tokens_details, "image_tokens", None), + cached_tokens=0, + ) + + normalized_usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + prompt_tokens_details=prompt_tokens_details, + completion_tokens_details=CompletionTokensDetailsWrapper( + text_tokens=0, + image_tokens=completion_tokens, + reasoning_tokens=0, + audio_tokens=0, + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=normalized_usage, + custom_llm_provider="vertex_ai", + ) + return prompt_cost + completion_cost def cost_calculator( @@ -18,6 +81,12 @@ def cost_calculator( custom_llm_provider="vertex_ai", ) + token_based_cost = _calculate_token_based_cost( + model=model, image_response=image_response + ) + if token_based_cost is not None: + return token_based_cost + output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if image_response.data: diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 9abce33fcd0..7e8848be301 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -9,9 +9,19 @@ import litellm from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) +from litellm.llms.gemini.image_generation.cost_calculator import ( + cost_calculator as gemini_image_generation_cost_calculator, +) +from litellm.llms.vertex_ai.image_generation.cost_calculator import ( + cost_calculator as vertex_image_generation_cost_calculator, +) from litellm.types.llms.openai import FileSearchTool, WebSearchOptions from litellm.types.utils import ( CompletionTokensDetailsWrapper, + ImageObject, + ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, ModelInfo, ModelResponse, PromptTokensDetailsWrapper, @@ -837,6 +847,140 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): ) +def test_vertex_image_generation_cost_prefers_token_usage_metadata(): + """ + When usage metadata exists on image responses, Vertex image generation cost + should be calculated from token pricing, not flat output_cost_per_image. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini-3.1-flash-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") + + input_text_tokens = 50 + input_image_tokens = 1120 + output_image_tokens = 1120 + prompt_tokens = input_text_tokens + input_image_tokens + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")], + usage=ImageUsage( + input_tokens=prompt_tokens, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=input_text_tokens, + image_tokens=input_image_tokens, + ), + output_tokens=output_image_tokens, + total_tokens=prompt_tokens + output_image_tokens, + ), + ) + + cost = vertex_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_prompt_cost = prompt_tokens * model_info["input_cost_per_token"] + expected_completion_cost = output_image_tokens * model_info["output_cost_per_image_token"] + expected_total_cost = expected_prompt_cost + expected_completion_cost + + assert round(cost, 10) == round(expected_total_cost, 10) + # Ensure this is not falling back to flat per-image pricing. + assert cost != len(image_response.data) * model_info["output_cost_per_image"] + + +def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(): + """ + Without usage metadata, Vertex image generation cost should fall back to + output_cost_per_image * number_of_images. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini-3.1-flash-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] + ) + + cost = vertex_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_cost = len(image_response.data) * model_info["output_cost_per_image"] + assert round(cost, 10) == round(expected_cost, 10) + + +def test_gemini_image_generation_cost_prefers_token_usage_metadata(): + """ + When usage metadata exists on image responses, Gemini image generation cost + should be calculated from token pricing, not flat output_cost_per_image. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini/gemini-3-pro-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + input_text_tokens = 20 + input_image_tokens = 1120 + output_image_tokens = 1120 + prompt_tokens = input_text_tokens + input_image_tokens + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")], + usage=ImageUsage( + input_tokens=prompt_tokens, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=input_text_tokens, + image_tokens=input_image_tokens, + ), + output_tokens=output_image_tokens, + total_tokens=prompt_tokens + output_image_tokens, + ), + ) + + cost = gemini_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_prompt_cost = prompt_tokens * model_info["input_cost_per_token"] + expected_completion_cost = output_image_tokens * model_info["output_cost_per_image_token"] + expected_total_cost = expected_prompt_cost + expected_completion_cost + + assert round(cost, 10) == round(expected_total_cost, 10) + # Ensure this is not falling back to flat per-image pricing. + assert cost != len(image_response.data) * model_info["output_cost_per_image"] + + +def test_gemini_image_generation_cost_falls_back_to_flat_image_pricing(): + """ + Without usage metadata, Gemini image generation cost should fall back to + output_cost_per_image * number_of_images. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini/gemini-3-pro-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] + ) + + cost = gemini_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_cost = len(image_response.data) * model_info["output_cost_per_image"] + assert round(cost, 10) == round(expected_cost, 10) + + def test_bedrock_anthropic_prompt_caching(): """Test Bedrock Anthropic models with prompt caching return correct costs.""" model = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" From 0e014253d7670df86cba872ffe0b87930a45b0a7 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Thu, 26 Feb 2026 15:07:51 -0600 Subject: [PATCH 14/46] refactor(cost): dedupe image token usage cost helper - extract shared calculate_image_response_cost_from_usage() helper\n- reuse helper in vertex and gemini image generation cost calculators\n- preserve provider-specific fallback to output_cost_per_image --- .../litellm_core_utils/llm_cost_calc/utils.py | 60 +++++++++++++++ .../image_generation/cost_calculator.py | 74 +++---------------- .../image_generation/cost_calculator.py | 72 ++---------------- 3 files changed, 76 insertions(+), 130 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index a9fd0f4ea8a..bf0b2709365 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -8,9 +8,11 @@ from litellm._logging import verbose_logger from litellm.types.utils import ( CacheCreationTokenDetails, CallTypes, + CompletionTokensDetailsWrapper, ImageResponse, ModelInfo, PassthroughCallTypes, + PromptTokensDetailsWrapper, ServiceTier, Usage, ) @@ -767,6 +769,64 @@ def generic_cost_per_token( # noqa: PLR0915 return prompt_cost, completion_cost +def calculate_image_response_cost_from_usage( + model: str, + image_response: ImageResponse, + custom_llm_provider: str, +) -> Optional[float]: + """ + Calculate image generation cost from usage metadata when available. + + Returns: + Optional[float]: total cost from token usage, or None when usage metadata + is missing/incomplete and caller should fall back to flat per-image pricing. + """ + usage = image_response.usage + if usage is None: + return None + + prompt_tokens = usage.input_tokens + completion_tokens = usage.output_tokens + total_tokens = usage.total_tokens + + if prompt_tokens is None or completion_tokens is None or total_tokens is None: + return None + + # ImageResponse may carry a default zeroed usage object even when provider + # usage metadata is absent. Treat this as missing usage and fall back. + if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: + return None + + input_tokens_details = getattr(usage, "input_tokens_details", None) + prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None + if input_tokens_details is not None: + prompt_tokens_details = PromptTokensDetailsWrapper( + text_tokens=getattr(input_tokens_details, "text_tokens", None), + image_tokens=getattr(input_tokens_details, "image_tokens", None), + cached_tokens=0, + ) + + normalized_usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + prompt_tokens_details=prompt_tokens_details, + completion_tokens_details=CompletionTokensDetailsWrapper( + text_tokens=0, + image_tokens=completion_tokens, + reasoning_tokens=0, + audio_tokens=0, + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=normalized_usage, + custom_llm_provider=custom_llm_provider, + ) + return prompt_cost + completion_cost + + class CostCalculatorUtils: @staticmethod def _call_type_has_image_response(call_type: str) -> bool: diff --git a/litellm/llms/gemini/image_generation/cost_calculator.py b/litellm/llms/gemini/image_generation/cost_calculator.py index 6d7572d5522..941ab0d50f7 100644 --- a/litellm/llms/gemini/image_generation/cost_calculator.py +++ b/litellm/llms/gemini/image_generation/cost_calculator.py @@ -2,71 +2,13 @@ Google AI Image Generation Cost Calculator """ -from typing import Any, Optional +from typing import Any import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token -from litellm.types.utils import ( - CompletionTokensDetailsWrapper, - ImageResponse, - PromptTokensDetailsWrapper, - Usage, +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + calculate_image_response_cost_from_usage, ) - - -def _calculate_token_based_cost(model: str, image_response: ImageResponse) -> Optional[float]: - """ - Calculate token-based image generation cost when usage metadata is available. - - Falls back to None when usage metadata is missing/incomplete. - """ - usage = image_response.usage - if usage is None: - return None - - prompt_tokens = usage.input_tokens - completion_tokens = usage.output_tokens - total_tokens = usage.total_tokens - - if ( - prompt_tokens is None - or completion_tokens is None - or total_tokens is None - ): - return None - # ImageResponse may carry a default zeroed usage object even when provider - # usage metadata is absent. Treat this as missing usage and fall back. - if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: - return None - - input_tokens_details = getattr(usage, "input_tokens_details", None) - prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None - if input_tokens_details is not None: - prompt_tokens_details = PromptTokensDetailsWrapper( - text_tokens=getattr(input_tokens_details, "text_tokens", None), - image_tokens=getattr(input_tokens_details, "image_tokens", None), - cached_tokens=0, - ) - - normalized_usage = Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - prompt_tokens_details=prompt_tokens_details, - completion_tokens_details=CompletionTokensDetailsWrapper( - text_tokens=0, - image_tokens=completion_tokens, - reasoning_tokens=0, - audio_tokens=0, - ), - ) - - prompt_cost, completion_cost = generic_cost_per_token( - model=model, - usage=normalized_usage, - custom_llm_provider="gemini", - ) - return prompt_cost + completion_cost +from litellm.types.utils import ImageResponse def cost_calculator( @@ -74,7 +16,7 @@ def cost_calculator( image_response: Any, ) -> float: """ - Vertex AI Image Generation Cost Calculator + Google AI Image Generation Cost Calculator """ _model_info = litellm.get_model_info( model=model, @@ -82,8 +24,10 @@ def cost_calculator( ) if isinstance(image_response, ImageResponse): - token_based_cost = _calculate_token_based_cost( - model=model, image_response=image_response + token_based_cost = calculate_image_response_cost_from_usage( + model=model, + image_response=image_response, + custom_llm_provider="gemini", ) if token_based_cost is not None: return token_based_cost diff --git a/litellm/llms/vertex_ai/image_generation/cost_calculator.py b/litellm/llms/vertex_ai/image_generation/cost_calculator.py index ac587182f04..012de5498cb 100644 --- a/litellm/llms/vertex_ai/image_generation/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_generation/cost_calculator.py @@ -2,71 +2,11 @@ Vertex AI Image Generation Cost Calculator """ -from typing import Optional - import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token -from litellm.types.utils import ( - CompletionTokensDetailsWrapper, - ImageResponse, - PromptTokensDetailsWrapper, - Usage, +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + calculate_image_response_cost_from_usage, ) - - -def _calculate_token_based_cost(model: str, image_response: ImageResponse) -> Optional[float]: - """ - Calculate token-based image generation cost when usage metadata is available. - - Falls back to None when usage metadata is missing/incomplete. - """ - usage = image_response.usage - if usage is None: - return None - - prompt_tokens = usage.input_tokens - completion_tokens = usage.output_tokens - total_tokens = usage.total_tokens - - if ( - prompt_tokens is None - or completion_tokens is None - or total_tokens is None - ): - return None - # ImageResponse may carry a default zeroed usage object even when provider - # usage metadata is absent. Treat this as missing usage and fall back. - if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: - return None - - input_tokens_details = getattr(usage, "input_tokens_details", None) - prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None - if input_tokens_details is not None: - prompt_tokens_details = PromptTokensDetailsWrapper( - text_tokens=getattr(input_tokens_details, "text_tokens", None), - image_tokens=getattr(input_tokens_details, "image_tokens", None), - cached_tokens=0, - ) - - normalized_usage = Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - prompt_tokens_details=prompt_tokens_details, - completion_tokens_details=CompletionTokensDetailsWrapper( - text_tokens=0, - image_tokens=completion_tokens, - reasoning_tokens=0, - audio_tokens=0, - ), - ) - - prompt_cost, completion_cost = generic_cost_per_token( - model=model, - usage=normalized_usage, - custom_llm_provider="vertex_ai", - ) - return prompt_cost + completion_cost +from litellm.types.utils import ImageResponse def cost_calculator( @@ -81,8 +21,10 @@ def cost_calculator( custom_llm_provider="vertex_ai", ) - token_based_cost = _calculate_token_based_cost( - model=model, image_response=image_response + token_based_cost = calculate_image_response_cost_from_usage( + model=model, + image_response=image_response, + custom_llm_provider="vertex_ai", ) if token_based_cost is not None: return token_based_cost From 516b18fecaa49a39d08217d30c3f59faf149e871 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:07:51 -0800 Subject: [PATCH 15/46] [Feature] Access group CRUD: Add bidirectional sync for teams/keys When creating, updating, or deleting access groups, automatically keep team and key access_group_ids in sync with the access group's assigned_team_ids and assigned_key_ids. Includes transaction-based DB updates, cache patching, and handles out-of-sync data by unioning assigned_* fields with hasSome queries. Adds 12 new tests covering sync behavior across all three CRUD operations. Co-Authored-By: Claude Sonnet 4.6 --- .../access_group_endpoints.py | 363 +++++++++++++----- .../test_access_group_endpoints.py | 284 ++++++++++++++ 2 files changed, 560 insertions(+), 87 deletions(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 12aa748bbc3..c6958240d9a 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Set from fastapi import APIRouter, Depends, HTTPException, status @@ -94,6 +94,183 @@ async def _invalidate_cache_access_group(access_group_id: str) -> None: ) +# --------------------------------------------------------------------------- +# DB sync helpers (called inside a Prisma transaction) +# --------------------------------------------------------------------------- + + +async def _sync_add_access_group_to_teams( + tx, team_ids: List[str], access_group_id: str +) -> None: + """Add access_group_id to each team's access_group_ids (idempotent).""" + for team_id in team_ids: + team = await tx.litellm_teamtable.find_unique(where={"team_id": team_id}) + if team is not None and access_group_id not in (team.access_group_ids or []): + await tx.litellm_teamtable.update( + where={"team_id": team_id}, + data={"access_group_ids": list(team.access_group_ids or []) + [access_group_id]}, + ) + + +async def _sync_remove_access_group_from_teams( + tx, team_ids: List[str], access_group_id: str +) -> None: + """Remove access_group_id from each team's access_group_ids (idempotent).""" + for team_id in team_ids: + team = await tx.litellm_teamtable.find_unique(where={"team_id": team_id}) + if team is not None and access_group_id in (team.access_group_ids or []): + await tx.litellm_teamtable.update( + where={"team_id": team_id}, + data={"access_group_ids": [ag for ag in team.access_group_ids if ag != access_group_id]}, + ) + + +async def _sync_add_access_group_to_keys( + tx, key_tokens: List[str], access_group_id: str +) -> None: + """Add access_group_id to each key's access_group_ids (idempotent).""" + for token in key_tokens: + key = await tx.litellm_verificationtoken.find_unique(where={"token": token}) + if key is not None and access_group_id not in (key.access_group_ids or []): + await tx.litellm_verificationtoken.update( + where={"token": token}, + data={"access_group_ids": list(key.access_group_ids or []) + [access_group_id]}, + ) + + +async def _sync_remove_access_group_from_keys( + tx, key_tokens: List[str], access_group_id: str +) -> None: + """Remove access_group_id from each key's access_group_ids (idempotent).""" + for token in key_tokens: + key = await tx.litellm_verificationtoken.find_unique(where={"token": token}) + if key is not None and access_group_id in (key.access_group_ids or []): + await tx.litellm_verificationtoken.update( + where={"token": token}, + data={"access_group_ids": [ag for ag in key.access_group_ids if ag != access_group_id]}, + ) + + +# --------------------------------------------------------------------------- +# Cache patch helpers +# --------------------------------------------------------------------------- + + +async def _patch_team_caches_add_access_group( + team_ids: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached team objects to include access_group_id.""" + for team_id in team_ids: + cached_team = await _get_team_object_from_cache( + key="team_id:{}".format(team_id), + proxy_logging_obj=proxy_logging_obj, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + ) + if cached_team is None: + continue + if cached_team.access_group_ids is None: + cached_team.access_group_ids = [access_group_id] + elif access_group_id not in cached_team.access_group_ids: + cached_team.access_group_ids = list(cached_team.access_group_ids) + [access_group_id] + else: + continue + await _cache_team_object( + team_id=team_id, + team_table=cached_team, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +async def _patch_team_caches_remove_access_group( + team_ids: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached team objects to remove access_group_id.""" + for team_id in team_ids: + cached_team = await _get_team_object_from_cache( + key="team_id:{}".format(team_id), + proxy_logging_obj=proxy_logging_obj, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + ) + if cached_team is not None and cached_team.access_group_ids: + cached_team.access_group_ids = [ + ag for ag in cached_team.access_group_ids if ag != access_group_id + ] + await _cache_team_object( + team_id=team_id, + team_table=cached_team, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +async def _patch_key_caches_add_access_group( + key_tokens: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached key objects to include access_group_id.""" + for token in key_tokens: + cached_key = await user_api_key_cache.async_get_cache(key=token) + if cached_key is None: + continue + if isinstance(cached_key, dict): + cached_key = UserAPIKeyAuth(**cached_key) + if not isinstance(cached_key, UserAPIKeyAuth): + continue + if cached_key.access_group_ids is None: + cached_key.access_group_ids = [access_group_id] + elif access_group_id not in cached_key.access_group_ids: + cached_key.access_group_ids = list(cached_key.access_group_ids) + [access_group_id] + else: + continue + await _cache_key_object( + hashed_token=token, + user_api_key_obj=cached_key, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +async def _patch_key_caches_remove_access_group( + key_tokens: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached key objects to remove access_group_id.""" + for token in key_tokens: + cached_key = await user_api_key_cache.async_get_cache(key=token) + if cached_key is None: + continue + if isinstance(cached_key, dict): + cached_key = UserAPIKeyAuth(**cached_key) + if isinstance(cached_key, UserAPIKeyAuth) and cached_key.access_group_ids: + cached_key.access_group_ids = [ + ag for ag in cached_key.access_group_ids if ag != access_group_id + ] + await _cache_key_object( + hashed_token=token, + user_api_key_obj=cached_key, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +# --------------------------------------------------------------------------- +# CRUD endpoints +# --------------------------------------------------------------------------- + + @router.post( "/v1/access_group", response_model=AccessGroupResponse, @@ -106,32 +283,42 @@ async def create_access_group( _require_proxy_admin(user_api_key_dict) prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - existing = await prisma_client.db.litellm_accessgrouptable.find_unique( - where={"access_group_name": data.access_group_name} - ) - if existing is not None: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=f"Access group '{data.access_group_name}' already exists", - ) - try: - record = await prisma_client.db.litellm_accessgrouptable.create( - data={ - "access_group_name": data.access_group_name, - "description": data.description, - "access_model_names": data.access_model_names or [], - "access_mcp_server_ids": data.access_mcp_server_ids or [], - "access_agent_ids": data.access_agent_ids or [], - "assigned_team_ids": data.assigned_team_ids or [], - "assigned_key_ids": data.assigned_key_ids or [], - "created_by": user_api_key_dict.user_id, - "updated_by": user_api_key_dict.user_id, - } - ) + async with prisma_client.db.tx() as tx: + existing = await tx.litellm_accessgrouptable.find_unique( + where={"access_group_name": data.access_group_name} + ) + if existing is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Access group '{data.access_group_name}' already exists", + ) + + record = await tx.litellm_accessgrouptable.create( + data={ + "access_group_name": data.access_group_name, + "description": data.description, + "access_model_names": data.access_model_names or [], + "access_mcp_server_ids": data.access_mcp_server_ids or [], + "access_agent_ids": data.access_agent_ids or [], + "assigned_team_ids": data.assigned_team_ids or [], + "assigned_key_ids": data.assigned_key_ids or [], + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, + } + ) + + # Sync team and key tables to reference the new access group + await _sync_add_access_group_to_teams( + tx, data.assigned_team_ids or [], record.access_group_id + ) + await _sync_add_access_group_to_keys( + tx, data.assigned_key_ids or [], record.access_group_id + ) + except HTTPException: + raise except Exception as e: # Race condition: another request created the same name between find_unique and create. - # Prisma raises UniqueViolationError (P2002) or similar for unique constraint. if "unique constraint" in str(e).lower() or "P2002" in str(e): raise HTTPException( status_code=status.HTTP_409_CONFLICT, @@ -139,8 +326,15 @@ async def create_access_group( ) raise - # Cache the newly created access group for read-heavy access patterns + from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache + await _cache_access_group_record(record) + await _patch_team_caches_add_access_group( + data.assigned_team_ids or [], record.access_group_id, user_api_key_cache, proxy_logging_obj + ) + await _patch_key_caches_add_access_group( + data.assigned_key_ids or [], record.access_group_id, user_api_key_cache, proxy_logging_obj + ) return _record_to_response(record) @@ -204,15 +398,35 @@ async def update_access_group( detail=f"Access group '{access_group_id}' not found", ) + # Compute team/key assignment deltas before the transaction + update_fields = data.model_dump(exclude_unset=True) + old_team_ids: Set[str] = set(existing.assigned_team_ids or []) + old_key_ids: Set[str] = set(existing.assigned_key_ids or []) + new_team_ids: Set[str] = set(update_fields["assigned_team_ids"]) if "assigned_team_ids" in update_fields else old_team_ids + new_key_ids: Set[str] = set(update_fields["assigned_key_ids"]) if "assigned_key_ids" in update_fields else old_key_ids + + teams_to_add = list(new_team_ids - old_team_ids) + teams_to_remove = list(old_team_ids - new_team_ids) + keys_to_add = list(new_key_ids - old_key_ids) + keys_to_remove = list(old_key_ids - new_key_ids) + update_data: dict = {"updated_by": user_api_key_dict.user_id} - for field, value in data.model_dump(exclude_unset=True).items(): + for field, value in update_fields.items(): update_data[field] = value try: - record = await prisma_client.db.litellm_accessgrouptable.update( - where={"access_group_id": access_group_id}, - data=update_data, - ) + async with prisma_client.db.tx() as tx: + record = await tx.litellm_accessgrouptable.update( + where={"access_group_id": access_group_id}, + data=update_data, + ) + + await _sync_add_access_group_to_teams(tx, teams_to_add, access_group_id) + await _sync_remove_access_group_from_teams(tx, teams_to_remove, access_group_id) + await _sync_add_access_group_to_keys(tx, keys_to_add, access_group_id) + await _sync_remove_access_group_from_keys(tx, keys_to_remove, access_group_id) + except HTTPException: + raise except Exception as e: # Unique constraint violation (e.g. access_group_name already exists). if "unique constraint" in str(e).lower() or "P2002" in str(e): @@ -222,8 +436,13 @@ async def update_access_group( ) raise - # Write the updated record into cache (same key, overwrites stale entry) + from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache + await _cache_access_group_record(record) + await _patch_team_caches_add_access_group(teams_to_add, access_group_id, user_api_key_cache, proxy_logging_obj) + await _patch_team_caches_remove_access_group(teams_to_remove, access_group_id, user_api_key_cache, proxy_logging_obj) + await _patch_key_caches_add_access_group(keys_to_add, access_group_id, user_api_key_cache, proxy_logging_obj) + await _patch_key_caches_remove_access_group(keys_to_remove, access_group_id, user_api_key_cache, proxy_logging_obj) return _record_to_response(record) @@ -240,9 +459,8 @@ async def delete_access_group( prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) try: - # Track affected team IDs and key tokens for cache invalidation - affected_team_ids: list = [] - affected_key_tokens: list = [] + affected_team_ids: List[str] = [] + affected_key_tokens: List[str] = [] async with prisma_client.db.tx() as tx: existing = await tx.litellm_accessgrouptable.find_unique( @@ -254,73 +472,44 @@ async def delete_access_group( detail=f"Access group '{access_group_id}' not found", ) - # Remove access_group_id from teams and keys that reference it + # Union of: teams that have this access_group_id in their own access_group_ids + # AND teams listed in assigned_team_ids (handles out-of-sync data from before this sync was added) teams_with_group = await tx.litellm_teamtable.find_many( where={"access_group_ids": {"hasSome": [access_group_id]}} ) - for team in teams_with_group: - affected_team_ids.append(team.team_id) - updated_ids = [tid for tid in (team.access_group_ids or []) if tid != access_group_id] - await tx.litellm_teamtable.update( - where={"team_id": team.team_id}, - data={"access_group_ids": updated_ids}, - ) + all_affected_team_ids: Set[str] = ( + {team.team_id for team in teams_with_group} + | set(existing.assigned_team_ids or []) + ) + affected_team_ids = list(all_affected_team_ids) + # Union of: keys that have this access_group_id in their own access_group_ids + # AND keys listed in assigned_key_ids (handles out-of-sync data) keys_with_group = await tx.litellm_verificationtoken.find_many( where={"access_group_ids": {"hasSome": [access_group_id]}} ) - for key in keys_with_group: - affected_key_tokens.append(key.token) - updated_ids = [kid for kid in (key.access_group_ids or []) if kid != access_group_id] - await tx.litellm_verificationtoken.update( - where={"token": key.token}, - data={"access_group_ids": updated_ids}, - ) + all_affected_key_tokens: Set[str] = ( + {key.token for key in keys_with_group} + | set(existing.assigned_key_ids or []) + ) + affected_key_tokens = list(all_affected_key_tokens) + + await _sync_remove_access_group_from_teams(tx, affected_team_ids, access_group_id) + await _sync_remove_access_group_from_keys(tx, affected_key_tokens, access_group_id) await tx.litellm_accessgrouptable.delete( where={"access_group_id": access_group_id} ) - # Invalidate the deleted access group from cache - await _invalidate_cache_access_group(access_group_id) - - # Patch cached team and key objects to remove the deleted access_group_id - # instead of fully invalidating them (keeps cache warm, avoids DB re-fetch) from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache - for team_id in affected_team_ids: - cached_team = await _get_team_object_from_cache( - key="team_id:{}".format(team_id), - proxy_logging_obj=proxy_logging_obj, - user_api_key_cache=user_api_key_cache, - parent_otel_span=None, - ) - if cached_team is not None and cached_team.access_group_ids: - cached_team.access_group_ids = [ - ag_id for ag_id in cached_team.access_group_ids if ag_id != access_group_id - ] - await _cache_team_object( - team_id=team_id, - team_table=cached_team, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - - for token in affected_key_tokens: - cached_key = await user_api_key_cache.async_get_cache(key=token) - if cached_key is not None: - if isinstance(cached_key, dict): - cached_key = UserAPIKeyAuth(**cached_key) - if isinstance(cached_key, UserAPIKeyAuth) and cached_key.access_group_ids: - cached_key.access_group_ids = [ - ag_id for ag_id in cached_key.access_group_ids if ag_id != access_group_id - ] - await _cache_key_object( - hashed_token=token, - user_api_key_obj=cached_key, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + await _invalidate_cache_access_group(access_group_id) + await _patch_team_caches_remove_access_group( + affected_team_ids, access_group_id, user_api_key_cache, proxy_logging_obj + ) + await _patch_key_caches_remove_access_group( + affected_key_tokens, access_group_id, user_api_key_cache, proxy_logging_obj + ) except HTTPException: raise diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py index 9b6e0631762..a8842f7448b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -99,10 +99,12 @@ def client_and_mocks(monkeypatch): mock_team_table = MagicMock() mock_team_table.find_many = AsyncMock(return_value=[]) + mock_team_table.find_unique = AsyncMock(return_value=None) mock_team_table.update = AsyncMock(return_value=None) mock_key_table = MagicMock() mock_key_table.find_many = AsyncMock(return_value=[]) + mock_key_table.find_unique = AsyncMock(return_value=None) mock_key_table.update = AsyncMock(return_value=None) @asynccontextmanager @@ -570,11 +572,13 @@ def test_delete_access_group_cleans_up_teams_and_keys(client_and_mocks): team_with_group.team_id = "team-1" team_with_group.access_group_ids = ["ag-to-delete", "ag-other"] mock_team_table.find_many = AsyncMock(return_value=[team_with_group]) + mock_team_table.find_unique = AsyncMock(return_value=team_with_group) key_with_group = MagicMock() key_with_group.token = "key-token-1" key_with_group.access_group_ids = ["ag-to-delete"] mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + mock_key_table.find_unique = AsyncMock(return_value=key_with_group) resp = client.delete("/v1/access_group/ag-to-delete") assert resp.status_code == 204 @@ -669,11 +673,13 @@ def test_delete_access_group_patches_cached_team_and_key( team_with_group.team_id = "team-1" team_with_group.access_group_ids = ["ag-to-delete", "ag-keep"] mock_team_table.find_many = AsyncMock(return_value=[team_with_group]) + mock_team_table.find_unique = AsyncMock(return_value=team_with_group) key_with_group = MagicMock() key_with_group.token = "hashed-key-1" key_with_group.access_group_ids = ["ag-to-delete"] mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + mock_key_table.find_unique = AsyncMock(return_value=key_with_group) # Build cached team object (returned from proxy_logging dual cache) if team_cache_group_ids is not None: @@ -762,6 +768,7 @@ def test_delete_access_group_patches_key_cached_as_dict(client_and_mocks): key_with_group.token = "hashed-key-dict" key_with_group.access_group_ids = ["ag-to-delete", "ag-other"] mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + mock_key_table.find_unique = AsyncMock(return_value=key_with_group) # No team in cache mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock( @@ -882,3 +889,280 @@ def test_record_to_access_group_table(): assert result.access_group_name == "unit-test-group" assert result.access_model_names == ["gpt-4", "claude-3"] assert result.access_agent_ids == ["agent-1"] + + +# --------------------------------------------------------------------------- +# Sync tests: CREATE +# --------------------------------------------------------------------------- + + +def test_create_access_group_syncs_assigned_teams(client_and_mocks): + """Create adds access_group_id to each assigned team's access_group_ids in DB.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + team_record = MagicMock() + team_record.team_id = "team-1" + team_record.access_group_ids = [] + mock_team_table.find_unique = AsyncMock(return_value=team_record) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_team_ids": ["team-1"]}, + ) + assert resp.status_code == 201 + + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-1"}) + mock_team_table.update.assert_awaited_once() + call_kwargs = mock_team_table.update.call_args.kwargs + assert call_kwargs["where"] == {"team_id": "team-1"} + # The newly created access group id ("ag-new") should be in the updated list + assert "ag-new" in call_kwargs["data"]["access_group_ids"] + + +def test_create_access_group_syncs_assigned_keys(client_and_mocks): + """Create adds access_group_id to each assigned key's access_group_ids in DB.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + key_record = MagicMock() + key_record.token = "hashed-token-1" + key_record.access_group_ids = [] + mock_key_table.find_unique = AsyncMock(return_value=key_record) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_key_ids": ["hashed-token-1"]}, + ) + assert resp.status_code == 201 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "hashed-token-1"}) + mock_key_table.update.assert_awaited_once() + call_kwargs = mock_key_table.update.call_args.kwargs + assert call_kwargs["where"] == {"token": "hashed-token-1"} + assert "ag-new" in call_kwargs["data"]["access_group_ids"] + + +def test_create_access_group_skips_sync_for_nonexistent_team(client_and_mocks): + """Create skips updating a team that doesn't exist in DB.""" + client, mock_prisma, _, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + mock_team_table.find_unique = AsyncMock(return_value=None) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_team_ids": ["nonexistent-team"]}, + ) + assert resp.status_code == 201 + mock_team_table.update.assert_not_awaited() + + +def test_create_access_group_idempotent_team_sync(client_and_mocks): + """Create skips updating a team that already has the access_group_id.""" + client, mock_prisma, _, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + team_record = MagicMock() + team_record.team_id = "team-1" + team_record.access_group_ids = ["ag-new"] # already synced + mock_team_table.find_unique = AsyncMock(return_value=team_record) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_team_ids": ["team-1"]}, + ) + assert resp.status_code == 201 + mock_team_table.update.assert_not_awaited() + + +# --------------------------------------------------------------------------- +# Sync tests: UPDATE +# --------------------------------------------------------------------------- + + +def test_update_access_group_syncs_added_teams(client_and_mocks): + """Update adds access_group_id to newly assigned teams.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_team_ids=["team-existing"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + team_record = MagicMock() + team_record.team_id = "team-new" + team_record.access_group_ids = [] + mock_team_table.find_unique = AsyncMock(return_value=team_record) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_team_ids": ["team-existing", "team-new"]}, + ) + assert resp.status_code == 200 + + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-new"}) + mock_team_table.update.assert_awaited_once() + call_kwargs = mock_team_table.update.call_args.kwargs + assert call_kwargs["where"] == {"team_id": "team-new"} + assert "ag-update" in call_kwargs["data"]["access_group_ids"] + + +def test_update_access_group_syncs_removed_teams(client_and_mocks): + """Update removes access_group_id from de-assigned teams.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_team_ids=["team-keep", "team-remove"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + team_to_remove = MagicMock() + team_to_remove.team_id = "team-remove" + team_to_remove.access_group_ids = ["ag-update"] + mock_team_table.find_unique = AsyncMock(return_value=team_to_remove) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_team_ids": ["team-keep"]}, + ) + assert resp.status_code == 200 + + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-remove"}) + mock_team_table.update.assert_awaited_once() + call_kwargs = mock_team_table.update.call_args.kwargs + assert call_kwargs["where"] == {"team_id": "team-remove"} + assert "ag-update" not in call_kwargs["data"]["access_group_ids"] + + +def test_update_access_group_no_team_sync_when_ids_not_in_payload(client_and_mocks): + """Update does not sync teams when assigned_team_ids is absent from the payload.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_team_ids=["team-1"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + resp = client.put("/v1/access_group/ag-update", json={"description": "new desc"}) + assert resp.status_code == 200 + + mock_team_table.find_unique.assert_not_awaited() + mock_team_table.update.assert_not_awaited() + + +def test_update_access_group_syncs_added_keys(client_and_mocks): + """Update adds access_group_id to newly assigned keys.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_key_ids=["old-token"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + key_record = MagicMock() + key_record.token = "new-token" + key_record.access_group_ids = [] + mock_key_table.find_unique = AsyncMock(return_value=key_record) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_key_ids": ["old-token", "new-token"]}, + ) + assert resp.status_code == 200 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "new-token"}) + mock_key_table.update.assert_awaited_once() + call_kwargs = mock_key_table.update.call_args.kwargs + assert call_kwargs["where"] == {"token": "new-token"} + assert "ag-update" in call_kwargs["data"]["access_group_ids"] + + +def test_update_access_group_syncs_removed_keys(client_and_mocks): + """Update removes access_group_id from de-assigned keys.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_key_ids=["keep-token", "remove-token"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + key_to_remove = MagicMock() + key_to_remove.token = "remove-token" + key_to_remove.access_group_ids = ["ag-update"] + mock_key_table.find_unique = AsyncMock(return_value=key_to_remove) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_key_ids": ["keep-token"]}, + ) + assert resp.status_code == 200 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "remove-token"}) + mock_key_table.update.assert_awaited_once() + call_kwargs = mock_key_table.update.call_args.kwargs + assert call_kwargs["where"] == {"token": "remove-token"} + assert "ag-update" not in call_kwargs["data"]["access_group_ids"] + + +# --------------------------------------------------------------------------- +# Sync tests: DELETE (out-of-sync data handling) +# --------------------------------------------------------------------------- + + +def test_delete_access_group_handles_out_of_sync_assigned_teams(client_and_mocks): + """Delete includes teams from assigned_team_ids even when not found by hasSome query.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + # Access group has assigned_team_ids but the team's access_group_ids is not synced + existing = _make_access_group_record( + access_group_id="ag-to-delete", + assigned_team_ids=["team-out-of-sync"], + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + # hasSome query finds nothing (team's own access_group_ids is out of sync) + mock_team_table.find_many = AsyncMock(return_value=[]) + + out_of_sync_team = MagicMock() + out_of_sync_team.team_id = "team-out-of-sync" + out_of_sync_team.access_group_ids = [] # already clean, no update needed + mock_team_table.find_unique = AsyncMock(return_value=out_of_sync_team) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 204 + + # find_unique is called for the out-of-sync team (included via union with assigned_team_ids) + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-out-of-sync"}) + # No update needed since team's access_group_ids doesn't contain "ag-to-delete" + mock_team_table.update.assert_not_awaited() + + +def test_delete_access_group_handles_out_of_sync_assigned_keys(client_and_mocks): + """Delete includes keys from assigned_key_ids even when not found by hasSome query.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record( + access_group_id="ag-to-delete", + assigned_key_ids=["token-out-of-sync"], + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + mock_key_table.find_many = AsyncMock(return_value=[]) + + out_of_sync_key = MagicMock() + out_of_sync_key.token = "token-out-of-sync" + out_of_sync_key.access_group_ids = [] + mock_key_table.find_unique = AsyncMock(return_value=out_of_sync_key) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 204 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "token-out-of-sync"}) + mock_key_table.update.assert_not_awaited() From 2d9ba674ec0178d7e2a8bccd75872ae3c4972d6b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:14:54 -0800 Subject: [PATCH 16/46] fix: move update_access_group find_unique inside transaction Eliminates TOCTOU race where existing record was read outside the transaction, allowing a concurrent update to make delta computation stale. Delta is now computed atomically within the same transaction as the write. Co-Authored-By: Claude Sonnet 4.6 --- .../access_group_endpoints.py | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index c6958240d9a..9a6ff219d4f 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -389,33 +389,34 @@ async def update_access_group( _require_proxy_admin(user_api_key_dict) prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - existing = await prisma_client.db.litellm_accessgrouptable.find_unique( - where={"access_group_id": access_group_id} - ) - if existing is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Access group '{access_group_id}' not found", - ) - - # Compute team/key assignment deltas before the transaction update_fields = data.model_dump(exclude_unset=True) - old_team_ids: Set[str] = set(existing.assigned_team_ids or []) - old_key_ids: Set[str] = set(existing.assigned_key_ids or []) - new_team_ids: Set[str] = set(update_fields["assigned_team_ids"]) if "assigned_team_ids" in update_fields else old_team_ids - new_key_ids: Set[str] = set(update_fields["assigned_key_ids"]) if "assigned_key_ids" in update_fields else old_key_ids - - teams_to_add = list(new_team_ids - old_team_ids) - teams_to_remove = list(old_team_ids - new_team_ids) - keys_to_add = list(new_key_ids - old_key_ids) - keys_to_remove = list(old_key_ids - new_key_ids) - update_data: dict = {"updated_by": user_api_key_dict.user_id} for field, value in update_fields.items(): update_data[field] = value try: async with prisma_client.db.tx() as tx: + # Read inside the transaction so delta computation is consistent with the write, + # avoiding a TOCTOU race where a concurrent update could make deltas stale. + existing = await tx.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if existing is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + + old_team_ids: Set[str] = set(existing.assigned_team_ids or []) + old_key_ids: Set[str] = set(existing.assigned_key_ids or []) + new_team_ids: Set[str] = set(update_fields["assigned_team_ids"]) if "assigned_team_ids" in update_fields else old_team_ids + new_key_ids: Set[str] = set(update_fields["assigned_key_ids"]) if "assigned_key_ids" in update_fields else old_key_ids + + teams_to_add = list(new_team_ids - old_team_ids) + teams_to_remove = list(old_team_ids - new_team_ids) + keys_to_add = list(new_key_ids - old_key_ids) + keys_to_remove = list(old_key_ids - new_key_ids) + record = await tx.litellm_accessgrouptable.update( where={"access_group_id": access_group_id}, data=update_data, From d9af3216100fec406f4f8a2bd3d3c5a10bb40938 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 10:08:44 +0530 Subject: [PATCH 17/46] Fix free models working from UI --- .../src/components/add_model/handle_add_model_submit.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index 8fa5ffd56a2..1d8c980c5ae 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -38,10 +38,11 @@ export const prepareModelAddRequest = async (formValues: Record, ac litellmParamsObj["model"] = mapping.litellm_model; // Handle pricing conversion before processing other fields - if (formValues.input_cost_per_token) { + // Use explicit checks to allow 0 (zero cost models for budget bypass) + if (formValues.input_cost_per_token !== undefined && formValues.input_cost_per_token !== null && formValues.input_cost_per_token !== "") { formValues.input_cost_per_token = Number(formValues.input_cost_per_token) / 1000000; } - if (formValues.output_cost_per_token) { + if (formValues.output_cost_per_token !== undefined && formValues.output_cost_per_token !== null && formValues.output_cost_per_token !== "") { formValues.output_cost_per_token = Number(formValues.output_cost_per_token) / 1000000; } // Keep input_cost_per_second as is, no conversion needed @@ -116,7 +117,7 @@ export const prepareModelAddRequest = async (formValues: Record, ac // Handle the pricing fields else if (key === "input_cost_per_token" || key === "output_cost_per_token" || key === "input_cost_per_second") { - if (value) { + if (value !== undefined && value !== null && value !== "") { litellmParamsObj[key] = Number(value); } continue; From 57c5efc785c2a222f33ac2669c370492f25b5728 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:39:57 -0800 Subject: [PATCH 18/46] refactor: initialize delta vars before try block and avoid redundant find_unique on delete - Initialize teams_to_add/teams_to_remove/keys_to_add/keys_to_remove before the try block in update_access_group for defensive clarity - In delete_access_group, update teams/keys returned by find_many directly (data already fetched) and use _sync_remove only for out-of-sync entities not found by the hasSome query, eliminating N+1 find_unique calls Co-Authored-By: Claude Sonnet 4.6 --- .../access_group_endpoints.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 9a6ff219d4f..a4d0b1104f7 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -394,6 +394,13 @@ async def update_access_group( for field, value in update_fields.items(): update_data[field] = value + # Initialize delta lists before the try block so they remain accessible + # for cache updates after the transaction, even if an error path is added later. + teams_to_add: List[str] = [] + teams_to_remove: List[str] = [] + keys_to_add: List[str] = [] + keys_to_remove: List[str] = [] + try: async with prisma_client.db.tx() as tx: # Read inside the transaction so delta computation is consistent with the write, @@ -495,8 +502,25 @@ async def delete_access_group( ) affected_key_tokens = list(all_affected_key_tokens) - await _sync_remove_access_group_from_teams(tx, affected_team_ids, access_group_id) - await _sync_remove_access_group_from_keys(tx, affected_key_tokens, access_group_id) + # Update teams returned by find_many directly — we already have their data. + for team in teams_with_group: + await tx.litellm_teamtable.update( + where={"team_id": team.team_id}, + data={"access_group_ids": [ag for ag in (team.access_group_ids or []) if ag != access_group_id]}, + ) + # Use _sync_remove only for out-of-sync teams not found by the hasSome query. + out_of_sync_team_ids = set(existing.assigned_team_ids or []) - {t.team_id for t in teams_with_group} + await _sync_remove_access_group_from_teams(tx, list(out_of_sync_team_ids), access_group_id) + + # Update keys returned by find_many directly — we already have their data. + for key in keys_with_group: + await tx.litellm_verificationtoken.update( + where={"token": key.token}, + data={"access_group_ids": [ag for ag in (key.access_group_ids or []) if ag != access_group_id]}, + ) + # Use _sync_remove only for out-of-sync keys not found by the hasSome query. + out_of_sync_key_tokens = set(existing.assigned_key_ids or []) - {k.token for k in keys_with_group} + await _sync_remove_access_group_from_keys(tx, list(out_of_sync_key_tokens), access_group_id) await tx.litellm_accessgrouptable.delete( where={"access_group_id": access_group_id} From 577f7037698450d393b115c715d785f8e7ebbcb4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 10:12:03 +0530 Subject: [PATCH 19/46] Register custom pricing in litellm.model_cost --- litellm/router.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index cbe5b414040..d89a5099b01 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6677,6 +6677,22 @@ class Router: # initialize client self._add_deployment(deployment=deployment) + # Register custom pricing in litellm.model_cost. + # Mirrors _create_deployment() logic to ensure dynamically-added deployments + # (e.g., loaded from DB) also have their custom pricing registered. + # Without this, _is_model_cost_zero() cannot detect explicitly-configured + # zero-cost models, causing budget checks to block free models. + _model_id = deployment.model_info.id + if _model_id is not None: + _model_info_dict: dict = deployment.model_info.model_dump( + exclude_none=True + ) + for field in CustomPricingLiteLLMParams.model_fields.keys(): + field_value = deployment.litellm_params.get(field) + if field_value is not None: + _model_info_dict[field] = field_value + litellm.register_model(model_cost={_model_id: _model_info_dict}) + # add to model names self._add_model_to_list_and_index_map( model=_deployment, model_id=deployment.model_info.id From 2144e79bada00324025b0bd2f5083bd6fd644c10 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:48:04 -0800 Subject: [PATCH 20/46] fix: guard against null assigned_*_ids in update_access_group delta computation set(None) raises TypeError when a client sends null for assigned_team_ids or assigned_key_ids. Add `or []` to handle null safely, consistent with create. Add test covering this case. Co-Authored-By: Claude Sonnet 4.6 --- .../access_group_endpoints.py | 4 ++-- .../test_access_group_endpoints.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index a4d0b1104f7..7e75060e87e 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -416,8 +416,8 @@ async def update_access_group( old_team_ids: Set[str] = set(existing.assigned_team_ids or []) old_key_ids: Set[str] = set(existing.assigned_key_ids or []) - new_team_ids: Set[str] = set(update_fields["assigned_team_ids"]) if "assigned_team_ids" in update_fields else old_team_ids - new_key_ids: Set[str] = set(update_fields["assigned_key_ids"]) if "assigned_key_ids" in update_fields else old_key_ids + new_team_ids: Set[str] = set(update_fields["assigned_team_ids"] or []) if "assigned_team_ids" in update_fields else old_team_ids + new_key_ids: Set[str] = set(update_fields["assigned_key_ids"] or []) if "assigned_key_ids" in update_fields else old_key_ids teams_to_add = list(new_team_ids - old_team_ids) teams_to_remove = list(old_team_ids - new_team_ids) diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py index a8842f7448b..fc3c87a112a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -1166,3 +1166,22 @@ def test_delete_access_group_handles_out_of_sync_assigned_keys(client_and_mocks) mock_key_table.find_unique.assert_awaited_once_with(where={"token": "token-out-of-sync"}) mock_key_table.update.assert_not_awaited() + + +def test_update_access_group_null_assigned_ids_treated_as_empty(client_and_mocks): + """Update with explicit null for assigned_*_ids clears the list without TypeError.""" + client, _, mock_table, *_ = client_and_mocks + + existing = _make_access_group_record( + access_group_id="ag-update", + assigned_team_ids=["team-1"], + assigned_key_ids=["key-1"], + ) + mock_table.find_unique = AsyncMock(return_value=existing) + + # Sending null for assigned_team_ids and assigned_key_ids + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_team_ids": None, "assigned_key_ids": None}, + ) + assert resp.status_code == 200 From cde23e9b6eaea89e43f07ef4b7393cd1a424e93b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 20:53:30 -0800 Subject: [PATCH 21/46] fix: normalize null list fields to [] in update_data before DB write When a client sends null for assigned_team_ids or assigned_key_ids, ensure the DB receives [] instead of null, preventing null from being stored where empty list is expected. Extend test to verify the DB call uses []. Co-Authored-By: Claude Sonnet 4.6 --- .../proxy/management_endpoints/access_group_endpoints.py | 2 ++ .../management_endpoints/test_access_group_endpoints.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 7e75060e87e..53dfbcda836 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -392,6 +392,8 @@ async def update_access_group( update_fields = data.model_dump(exclude_unset=True) update_data: dict = {"updated_by": user_api_key_dict.user_id} for field, value in update_fields.items(): + if field in ("assigned_team_ids", "assigned_key_ids") and value is None: + value = [] update_data[field] = value # Initialize delta lists before the try block so they remain accessible diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py index fc3c87a112a..32fd0750de8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -1169,7 +1169,7 @@ def test_delete_access_group_handles_out_of_sync_assigned_keys(client_and_mocks) def test_update_access_group_null_assigned_ids_treated_as_empty(client_and_mocks): - """Update with explicit null for assigned_*_ids clears the list without TypeError.""" + """Update with explicit null for assigned_*_ids clears the list and writes [] to DB.""" client, _, mock_table, *_ = client_and_mocks existing = _make_access_group_record( @@ -1185,3 +1185,8 @@ def test_update_access_group_null_assigned_ids_treated_as_empty(client_and_mocks json={"assigned_team_ids": None, "assigned_key_ids": None}, ) assert resp.status_code == 200 + + # Verify the DB update was called with [] (not null) for list fields + update_call_kwargs = mock_table.update.call_args.kwargs + assert update_call_kwargs["data"]["assigned_team_ids"] == [] + assert update_call_kwargs["data"]["assigned_key_ids"] == [] From aaf0570f225201f82fe7236355aae70c6d59bcf2 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 26 Feb 2026 21:02:12 -0800 Subject: [PATCH 22/46] fix: normalize null to [] for all Optional[List[str]] fields in update_data Extend the null normalization to access_model_names, access_mcp_server_ids, and access_agent_ids in addition to assigned_team_ids and assigned_key_ids. Writing null for non-optional list fields causes ValidationError on read. Co-Authored-By: Claude Sonnet 4.6 --- litellm/proxy/management_endpoints/access_group_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 53dfbcda836..d58dca5aec0 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -392,7 +392,7 @@ async def update_access_group( update_fields = data.model_dump(exclude_unset=True) update_data: dict = {"updated_by": user_api_key_dict.user_id} for field, value in update_fields.items(): - if field in ("assigned_team_ids", "assigned_key_ids") and value is None: + if field in ("assigned_team_ids", "assigned_key_ids", "access_model_names", "access_mcp_server_ids", "access_agent_ids") and value is None: value = [] update_data[field] = value From 6b9ec4247f51faf42da893e1cc7fe952d2cdface Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 11:31:45 +0530 Subject: [PATCH 23/46] Preserve forwarding server side called tools --- .../prompt_templates/factory.py | 27 ++- tests/llm_translation/test_prompt_factory.py | 175 ++++++++++++++++++ 2 files changed, 194 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index ba415af9a5a..796223ff8e1 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1766,6 +1766,7 @@ def convert_function_to_anthropic_tool_invoke( def convert_to_anthropic_tool_invoke( tool_calls: List[ChatCompletionAssistantToolCall], web_search_results: Optional[List[Any]] = None, + tool_results: Optional[List[Any]] = None, ) -> List[Union[AnthropicMessagesToolUseParam, Dict[str, Any]]]: """ OpenAI tool invokes: @@ -1840,12 +1841,18 @@ def convert_to_anthropic_tool_invoke( } anthropic_tool_invoke.append(_anthropic_server_tool_use) - # Add corresponding web_search_tool_result if available + # Add corresponding tool result if available. + # Check both web_search_results (web_search_tool_result / web_fetch_tool_result) + # and tool_results (bash_code_execution_tool_result, etc.) + _all_tool_results: List[Any] = [] if web_search_results: - for result in web_search_results: - if result.get("tool_use_id") == tool_id: - anthropic_tool_invoke.append(result) - break + _all_tool_results.extend(web_search_results) + if tool_results: + _all_tool_results.extend(tool_results) + for result in _all_tool_results: + if result.get("tool_use_id") == tool_id: + anthropic_tool_invoke.append(result) + break else: # Regular tool_use sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id) @@ -2472,9 +2479,10 @@ def anthropic_messages_pt( # noqa: PLR0915 # Pass through as-is since these are Anthropic-native content types elif m.get("type", "") == "server_tool_use": assistant_content.append(m) # type: ignore - # handle tool_search_tool_result blocks + # handle all *_tool_result blocks (tool_search_tool_result, + # web_search_tool_result, bash_code_execution_tool_result, etc.) # Pass through as-is since these are Anthropic-native content types - elif m.get("type", "") == "tool_search_tool_result": + elif m.get("type", "").endswith("_tool_result"): assistant_content.append(m) # type: ignore elif ( "content" in assistant_content_block @@ -2504,7 +2512,8 @@ def anthropic_messages_pt( # noqa: PLR0915 if ( assistant_tool_calls is not None ): # support assistant tool invoke conversion - # Get web_search_results from provider_specific_fields for server_tool_use reconstruction + # Get web_search_results and tool_results from provider_specific_fields + # for server_tool_use reconstruction. # Fixes: https://github.com/BerriAI/litellm/issues/17737 _provider_specific_fields_raw = assistant_content_block.get( "provider_specific_fields" @@ -2517,9 +2526,11 @@ def anthropic_messages_pt( # noqa: PLR0915 _web_search_results = _provider_specific_fields.get( "web_search_results" ) + _tool_results = _provider_specific_fields.get("tool_results") tool_invoke_results = convert_to_anthropic_tool_invoke( assistant_tool_calls, web_search_results=_web_search_results, + tool_results=_tool_results, ) # Prevent "tool_use ids must be unique" errors by filtering duplicates diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 88bca007740..9f902f2bd86 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -1214,6 +1214,181 @@ def test_anthropic_messages_pt_with_server_tool_use(): assert tool_use["id"] == "toolu_01XYZ789" +def test_convert_to_anthropic_tool_invoke_with_tool_results(): + """ + Test that non-web-search *_tool_result blocks (e.g. bash_code_execution_tool_result) + stored in provider_specific_fields["tool_results"] are paired with their server_tool_use + block when reconstructing assistant history. + + Regression for: server tool result blocks dropped on multi-turn replay + (bash_code_execution_tool_result, text_editor_code_execution_tool_result, etc.) + """ + tool_calls = [ + { + "id": "srvtoolu_01BASH", + "type": "function", + "function": { + "name": "bash_code_execution", + "arguments": '{"command": "python3 -c \\"print(2)\\""}', + }, + } + ] + + tool_results = [ + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01BASH", + "content": { + "type": "bash_code_execution_result", + "stdout": "2\n", + "stderr": "", + "return_code": 0, + "content": [], + }, + } + ] + + result = convert_to_anthropic_tool_invoke(tool_calls, tool_results=tool_results) + + assert len(result) == 2 + # First: server_tool_use + assert result[0]["type"] == "server_tool_use" + assert result[0]["id"] == "srvtoolu_01BASH" + assert result[0]["name"] == "bash_code_execution" + # Second: bash_code_execution_tool_result paired correctly + assert result[1]["type"] == "bash_code_execution_tool_result" + assert result[1]["tool_use_id"] == "srvtoolu_01BASH" + + +def test_anthropic_messages_pt_raw_bash_tool_result_passthrough(): + """ + Test that raw assistant content lists containing bash_code_execution_tool_result + blocks are passed through intact to Anthropic. + + Regression: the raw-block passthrough only handled tool_search_tool_result; + bash_code_execution_tool_result and other *_tool_result types were silently dropped. + """ + messages = [ + {"role": "user", "content": "What is 1+1?"}, + { + "role": "assistant", + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_01BASH", + "name": "bash_code_execution", + "input": {"command": "python3 -c \"print(1+1)\""}, + }, + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01BASH", + "content": { + "type": "bash_code_execution_result", + "stdout": "2\n", + "stderr": "", + "return_code": 0, + "content": [], + }, + }, + {"type": "text", "text": "The answer is 2."}, + ], + }, + {"role": "user", "content": "Thanks!"}, + ] + + result = anthropic_messages_pt( + messages, model="claude-sonnet-4-5", llm_provider="anthropic" + ) + + assistant_msg = next(m for m in result if m["role"] == "assistant") + content = assistant_msg["content"] + types = [c.get("type") for c in content] + + assert "server_tool_use" in types, "server_tool_use block must be preserved" + assert ( + "bash_code_execution_tool_result" in types + ), "bash_code_execution_tool_result block must not be dropped" + assert "text" in types + + # Result must immediately follow its server_tool_use + srv_idx = types.index("server_tool_use") + result_idx = types.index("bash_code_execution_tool_result") + assert result_idx == srv_idx + 1 + + bash_result = next( + c for c in content if c.get("type") == "bash_code_execution_tool_result" + ) + assert bash_result["tool_use_id"] == "srvtoolu_01BASH" + + +def test_anthropic_messages_pt_with_bash_tool_result_in_provider_specific_fields(): + """ + Test that anthropic_messages_pt correctly reconstructs bash_code_execution_tool_result + from provider_specific_fields["tool_results"] when replaying LiteLLM response objects. + + Regression: only web_search_results were read from provider_specific_fields; + tool_results (bash_code_execution_tool_result, etc.) were silently lost. + """ + messages = [ + {"role": "user", "content": "What is 1+1?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "srvtoolu_01BASH", + "type": "function", + "function": { + "name": "bash_code_execution", + "arguments": '{"command": "python3 -c \\"print(1+1)\\""}', + }, + } + ], + "provider_specific_fields": { + "tool_results": [ + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01BASH", + "content": { + "type": "bash_code_execution_result", + "stdout": "2\n", + "stderr": "", + "return_code": 0, + "content": [], + }, + } + ] + }, + }, + {"role": "user", "content": "Thanks!"}, + ] + + result = anthropic_messages_pt( + messages, model="claude-sonnet-4-5", llm_provider="anthropic" + ) + + assistant_msg = next(m for m in result if m["role"] == "assistant") + content = assistant_msg["content"] + types = [c.get("type") for c in content] + + assert "server_tool_use" in types, "server_tool_use block must be reconstructed" + assert ( + "bash_code_execution_tool_result" in types + ), "bash_code_execution_tool_result must be paired from provider_specific_fields['tool_results']" + + # Result must immediately follow its server_tool_use + srv_idx = types.index("server_tool_use") + result_idx = types.index("bash_code_execution_tool_result") + assert result_idx == srv_idx + 1 + + srv = next(c for c in content if c.get("type") == "server_tool_use") + assert srv["id"] == "srvtoolu_01BASH" + bash_result = next( + c for c in content if c.get("type") == "bash_code_execution_tool_result" + ) + assert bash_result["tool_use_id"] == "srvtoolu_01BASH" + + # ============ parse_tool_call_arguments Tests ============ # Tests for the shared utility that parses tool call JSON arguments From 596437b3b901db2f2b01e8a134d308fbaf46b4c4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 12:04:12 +0530 Subject: [PATCH 24/46] Add Regression tests for image_url blocks in assistant message content. --- .../types/llms/test_types_llms_openai.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/test_litellm/types/llms/test_types_llms_openai.py b/tests/test_litellm/types/llms/test_types_llms_openai.py index 87cc9586665..054fe505764 100644 --- a/tests/test_litellm/types/llms/test_types_llms_openai.py +++ b/tests/test_litellm/types/llms/test_types_llms_openai.py @@ -169,3 +169,97 @@ class TestResponsesAPIResponseOutputText: ) assert response.output_text == "" + + +class TestAssistantMessageImageUrlContent: + """ + Regression tests for image_url blocks in assistant message content. + + Bug: ChatCompletionAssistantMessage.content did not include + ChatCompletionImageObject in its union, so Pydantic v2 silently dropped + image_url blocks (content → []) when serialising via AllMessageValues. + This affects users who store conversation history as JSON (e.g. in a DB) + and read it back typed as list[AllMessageValues]. + """ + + ASSISTANT_MESSAGE_WITH_IMAGE = { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here is the image you requested:"}, + { + "type": "image_url", + "image_url": { + "url": ( + "data:image/png;base64," + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAA" + "DUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + ) + }, + }, + ], + } + + def test_assistant_message_image_url_preserved_single(self): + """ + TypeAdapter(ChatCompletionAssistantMessage): image_url block must survive + validate_python → dump_python without being dropped or raising an error. + """ + from typing import List + + from pydantic import TypeAdapter + + from litellm.types.llms.openai import ChatCompletionAssistantMessage + + adapter = TypeAdapter(ChatCompletionAssistantMessage) + validated = adapter.validate_python(self.ASSISTANT_MESSAGE_WITH_IMAGE) + dumped = adapter.dump_python(validated) + + raw_content = dumped.get("content") + # Pydantic may return a lazy SerializationIterator for Iterable fields; + # convert to list to consume it — this must not raise ValidationError. + content_blocks = list(raw_content) if raw_content is not None else [] + + assert len(content_blocks) == 2, ( + f"Expected 2 content blocks (text + image_url), got {len(content_blocks)}: {content_blocks}" + ) + types = [b.get("type") for b in content_blocks if isinstance(b, dict)] + assert "image_url" in types, f"image_url block was silently dropped; blocks: {content_blocks}" + + def test_assistant_message_image_url_preserved_in_all_message_values(self): + """ + TypeAdapter(List[AllMessageValues]) DB round-trip: image_url blocks in an + assistant message must not be silently dropped during dump_python(mode='json'). + + This is the primary failing path: conversation history stored as JSON in a + database and read back typed as list[AllMessageValues]. + """ + from typing import List + + from pydantic import TypeAdapter + + from litellm.types.llms.openai import AllMessageValues + + conversation = [ + { + "role": "user", + "content": "Generate an image of a banana wearing a LiteLLM costume", + }, + self.ASSISTANT_MESSAGE_WITH_IMAGE, + ] + + adapter = TypeAdapter(List[AllMessageValues]) + validated = adapter.validate_python(conversation) + dumped = adapter.dump_python(validated, mode="json") + + assistant = next((m for m in dumped if m.get("role") == "assistant"), None) + assert assistant is not None, "Assistant message missing after serialisation" + + content = assistant.get("content", []) + assert isinstance(content, list), f"content should be a list, got {type(content)}" + assert len(content) == 2, ( + f"Expected 2 content blocks (text + image_url), got {len(content)}: {content}" + ) + types = [b.get("type") for b in content if isinstance(b, dict)] + assert "image_url" in types, ( + f"image_url block was silently dropped during AllMessageValues serialisation; blocks: {content}" + ) From d0445e1e33590963018f0be4b644c4f4905fae0a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 13:00:57 +0530 Subject: [PATCH 25/46] Fix converse handling for parallel_tool_calls --- .../bedrock/chat/converse_transformation.py | 6 +- .../chat/test_converse_transformation.py | 95 +++++++++++-------- 2 files changed, 60 insertions(+), 41 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d4fd0606302..a0f2f65fb7f 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1217,15 +1217,15 @@ class AmazonConverseConfig(BaseConfig): # Handle parallel_tool_calls configuration parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None) - if parallel_tool_use_config is not None: - # Merge the tool_choice config from parallel_tool_calls into additional_request_params + if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model): for key, value in parallel_tool_use_config.items(): if key in additional_request_params and isinstance(additional_request_params[key], dict) and isinstance(value, dict): - # Merge dictionaries additional_request_params[key].update(value) else: additional_request_params[key] = value + additional_request_params.pop("parallel_tool_calls", None) + # Only set the topK value in for models that support it additional_request_params.update( self._handle_top_k_value(model, inference_params) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 26395597166..2b996977d8d 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -3382,59 +3382,78 @@ def test_output_config_applies_additional_properties(): -def test_parallel_tool_calls_in_request_transformation(): - """Test that parallel_tool_calls is correctly placed in additionalModelRequestFields after full transformation""" - config = AmazonConverseConfig() - - messages = [ - {"role": "user", "content": "What's the weather in SF and NYC?"} - ] - - non_default_params = { - "parallel_tool_calls": False, - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The location to get weather for" - } - }, - "required": ["location"] +_TOOL_PARAM = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The location to get weather for", } - } - } - ], - "max_tokens": 100, + }, + "required": ["location"], + }, + }, } - +] + + +def test_parallel_tool_calls_newer_model_adds_disable_flag(): + """Newer Claude models (4.5+) should get disable_parallel_tool_use in additionalModelRequestFields.""" + config = AmazonConverseConfig() + model = "anthropic.claude-sonnet-4-5-20250929-v1:0" + messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}] + optional_params = config.map_openai_params( - non_default_params=non_default_params, + non_default_params={"parallel_tool_calls": False, "tools": _TOOL_PARAM}, optional_params={}, - model="anthropic.claude-sonnet-4-5-v2:0", + model=model, drop_params=False, ) - - # Transform the request + request_data = config.transform_request( - model="anthropic.claude-sonnet-4-5-v2:0", + model=model, messages=messages, optional_params=optional_params, litellm_params={}, headers={}, ) - - # Verify the structure + assert "additionalModelRequestFields" in request_data assert "tool_choice" in request_data["additionalModelRequestFields"] - assert "disable_parallel_tool_use" in request_data["additionalModelRequestFields"]["tool_choice"] assert request_data["additionalModelRequestFields"]["tool_choice"]["disable_parallel_tool_use"] is True + assert "parallel_tool_calls" not in request_data["additionalModelRequestFields"] + + +def test_parallel_tool_calls_older_model_drops_disable_flag(): + """Older Claude models (pre-4.5) must NOT receive disable_parallel_tool_use — Bedrock rejects it.""" + config = AmazonConverseConfig() + model = "anthropic.claude-3-5-sonnet-20241022-v2:0" + messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}] + + optional_params = config.map_openai_params( + non_default_params={"parallel_tool_calls": False, "tools": _TOOL_PARAM}, + optional_params={}, + model=model, + drop_params=False, + ) + + request_data = config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + additional = request_data.get("additionalModelRequestFields", {}) + assert "tool_choice" not in additional + assert "parallel_tool_calls" not in additional class TestBedrockMinThinkingBudgetTokens: From 99c62ca40ea82ab31d61a251410d1a0367ae7aff Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 13:26:31 +0530 Subject: [PATCH 26/46] Add opt out varible for v1/messages to responses --- litellm/__init__.py | 3 +++ .../experimental_pass_through/messages/handler.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 6e42f2c1ea5..50fa0e76755 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -197,6 +197,9 @@ telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) +use_chat_completions_url_for_anthropic_messages: bool = bool( + os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) +) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API retry = True ### AUTH ### api_key: Optional[str] = None diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 6fe0fcd4fdf..5b215c1fe54 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -30,11 +30,17 @@ from .utils import AnthropicMessagesRequestUtils, mock_response # Providers that are routed directly to the OpenAI Responses API instead of # going through chat/completions. -_RESPONSES_API_PROVIDERS = frozenset({"openai", "azure", "azure_text"}) +_RESPONSES_API_PROVIDERS = frozenset({"openai"}) def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool: - """Return True when the provider should use the Responses API path.""" + """Return True when the provider should use the Responses API path. + + Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to + opt out and route OpenAI/Azure requests through chat/completions instead. + """ + if litellm.use_chat_completions_url_for_anthropic_messages: + return False return custom_llm_provider in _RESPONSES_API_PROVIDERS ####### ENVIRONMENT VARIABLES ################### From 2fa9b81e2fb5aae7ad8dc74663b6b5b3de6696c0 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 13:28:48 +0530 Subject: [PATCH 27/46] Add docs for opt out variable --- docs/my-website/docs/proxy/config_settings.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 8dbebad884e..4f862cf8471 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -196,6 +196,7 @@ router_settings: | disable_end_user_cost_tracking_prometheus_only | boolean | If true, turns off end user cost tracking on prometheus metrics only. | | key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) | | disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. | +| use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. | | disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). | | enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. | | disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. | From 3bec6f5a9a349a2b55273511e93c83f4247dd947 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 21:39:32 +0530 Subject: [PATCH 28/46] Fix: poetry lock --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 34227a69ccb..0314a360542 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3222,15 +3222,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.48" +version = "0.4.49" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.48-py3-none-any.whl", hash = "sha256:097001fccec5dbf4cffd902114898a9cfeba62673202447d55d2d0286cf93126"}, - {file = "litellm_proxy_extras-0.4.48.tar.gz", hash = "sha256:5d5d8acf31b92d0cd6738555fb4a2411819755155438de9fb23c724c356400a2"}, + {file = "litellm_proxy_extras-0.4.49-py3-none-any.whl", hash = "sha256:aeb0e08b4705c19fdc5b75a43c608a82fc36032f6d83be509dbf37baea62f2cd"}, + {file = "litellm_proxy_extras-0.4.49.tar.gz", hash = "sha256:d9bdae54d1e3398f2e2025c9d8b98a19e226874337d540d5415922d7dbbc97bb"}, ] [[package]] @@ -7989,4 +7989,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "b9b1e47b3b84748c0053be6a544c2399bf2601746a4f88dcb1be7c5e4eeab359" +content-hash = "bbc7d43f5484af4c8877fe66e34f8283069528379af49d573036ba144cc2eb7a" From d13508c1c56074a894ffd365b615037353c92d2b Mon Sep 17 00:00:00 2001 From: Noah Nistler <60981020+noahnistler@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:50:02 -0600 Subject: [PATCH 29/46] Enable local file support for OCR (#22133) * [Docs] Enable local file support Implemented internal handling for converting file-type documents to the required format for OCR processing, ensuring seamless integration with various providers. * Refactor OCR file handling and improve security checks Removed deprecated MIME type mapping and file conversion functions, replacing them with updated implementations. Enhanced security by rejecting 'file' document types in JSON requests, ensuring file uploads are handled via multipart/form-data. Updated tests to reflect these changes and ensure proper functionality. * Enhance MIME type validation in OCR processing Added a regular expression check to validate MIME types in the convert_file_document_to_url_document function, raising a ValueError for invalid types. Updated tests to ensure proper error handling for unsupported MIME types. * Enhance type safety in OCR file handling Added type casting for the uploaded file in the _parse_multipart_form function to ensure proper handling of UploadFile instances. This change improves type safety and reduces potential runtime errors during file processing. * Refactor MIME type handling in document uploads Updated the MIME type extraction logic to strip parameters from the Content-Type header, ensuring only the base type is used. Added tests to verify that MIME parameters are correctly handled and stripped in various scenarios. * Update OCR documentation for MIME type recommendations and remove unnecessary tips Clarified the recommended usage of MIME types for raw bytes in document uploads. Simplified the documentation by removing the tip about multipart file uploads from tools like Postman, ensuring a more concise and focused guide. * Enhance multipart form handling in OCR endpoints Updated the _parse_multipart_form function to ignore both 'file' and 'document' fields during form parsing, ensuring that the document built from the uploaded file is not overridden. Added a new test to verify that injected document fields do not affect the constructed document, improving security and robustness of the file upload process. --- docs/my-website/docs/ocr.md | 98 +++- litellm/llms/base_llm/ocr/transformation.py | 10 +- litellm/ocr/main.py | 226 +++++++-- litellm/proxy/ocr_endpoints/endpoints.py | 206 +++++++- tests/test_litellm/ocr/__init__.py | 0 tests/test_litellm/ocr/test_ocr_file_input.py | 464 ++++++++++++++++++ 6 files changed, 939 insertions(+), 65 deletions(-) create mode 100644 tests/test_litellm/ocr/__init__.py create mode 100644 tests/test_litellm/ocr/test_ocr_file_input.py diff --git a/docs/my-website/docs/ocr.md b/docs/my-website/docs/ocr.md index 93cb74ee69f..cea6fce1254 100644 --- a/docs/my-website/docs/ocr.md +++ b/docs/my-website/docs/ocr.md @@ -61,6 +61,52 @@ async def test_async_ocr(): asyncio.run(test_async_ocr()) ``` +### Using Local Files + +LiteLLM can read local files directly — no manual base64 encoding needed: + +```python +from litellm import ocr + +# OCR with a local PDF file path +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "file", + "file": "/path/to/document.pdf" + } +) + +# OCR with a file object +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "file", + "file": open("document.pdf", "rb") + } +) + +# OCR with raw bytes +with open("document.pdf", "rb") as f: + pdf_bytes = f.read() + +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "file", + "file": pdf_bytes, + "mime_type": "application/pdf" # recommended for raw bytes (auto-detected from extension for file paths) + } +) +``` + +The `file` field accepts: +- **File path** (`str` or `pathlib.Path`) — LiteLLM reads the file and detects the MIME type from the extension +- **File object** (binary file-like object) — e.g. `open("doc.pdf", "rb")` +- **Raw bytes** (`bytes`) — use `mime_type` to specify the content type + +LiteLLM automatically converts file inputs to base64 data URIs internally, so all providers work seamlessly. + ### Using Base64 Encoded Documents ```python @@ -121,7 +167,7 @@ litellm --config /path/to/config.yaml # RUNNING on http://0.0.0.0:4000 ``` -Test request +**Test request — JSON body** ```bash curl http://0.0.0.0:4000/v1/ocr \ @@ -136,6 +182,27 @@ curl http://0.0.0.0:4000/v1/ocr \ }' ``` +**Test request — multipart file upload** + +Upload a file directly using multipart form data. No need to base64-encode the file yourself. + +```bash +curl http://0.0.0.0:4000/v1/ocr \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@/path/to/document.pdf" +``` + +You can also pass optional parameters as additional form fields: + +```bash +curl http://0.0.0.0:4000/v1/ocr \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@screenshot.png" \ + -F 'pages=[0,1,2]' \ + -F "include_image_base64=true" +``` ## **Request/Response Format** @@ -168,10 +235,12 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie | Parameter | Type | Required | Description | |-----------|------|----------|-------------| | `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) | -| `document` | object | Yes | Document to process. Must contain `type` and URL field | -| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images | -| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) | -| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) | +| `document` | object | Yes | Document to process. Must contain `type` and the corresponding field | +| `document.type` | string | Yes | `"document_url"` for PDFs/docs, `"image_url"` for images, or `"file"` for local files | +| `document.document_url` | string | Conditional | URL or data URI to the document (required if `type` is `"document_url"`) | +| `document.image_url` | string | Conditional | URL or data URI to the image (required if `type` is `"image_url"`) | +| `document.file` | string/bytes/file | Conditional | File path, bytes, or file-like object (required if `type` is `"file"`) | +| `document.mime_type` | string | No | Explicit MIME type for file inputs (auto-detected from extension if not provided) | | `pages` | array | No | List of specific page indices to process (0-indexed) | | `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings | | `image_limit` | integer | No | Maximum number of images to return | @@ -179,7 +248,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie #### Document Format Examples -**For PDFs and documents:** +**For PDFs and documents (URL):** ```json { "type": "document_url", @@ -187,7 +256,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie } ``` -**For images:** +**For images (URL):** ```json { "type": "image_url", @@ -203,6 +272,21 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie } ``` +**For local files (SDK):** +```python +{"type": "file", "file": "/path/to/document.pdf"} +{"type": "file", "file": open("image.png", "rb")} +{"type": "file", "file": pdf_bytes, "mime_type": "application/pdf"} +``` + +**For file uploads (Proxy — multipart form):** +```bash +curl http://0.0.0.0:4000/v1/ocr \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@document.pdf" +``` + ### Response Format The response follows Mistral's OCR format with the following structure: diff --git a/litellm/llms/base_llm/ocr/transformation.py b/litellm/llms/base_llm/ocr/transformation.py index fb13332c464..29929a2bf62 100644 --- a/litellm/llms/base_llm/ocr/transformation.py +++ b/litellm/llms/base_llm/ocr/transformation.py @@ -15,7 +15,9 @@ else: LiteLLMLoggingObj = Any -# DocumentType for OCR - Mistral format document dict +# DocumentType for OCR - providers always receive a dict with +# type="document_url" or type="image_url" (str values only). +# File-type inputs are preprocessed to this format in litellm/ocr/main.py. DocumentType = Dict[str, str] @@ -141,9 +143,13 @@ class BaseOCRConfig: Transform OCR request to provider-specific format. Override in provider-specific implementations. + Note: By the time this method is called, any file-type documents have already + been converted to document_url/image_url format with base64 data URIs by + the preprocessing in litellm/ocr/main.py. + Args: model: Model name - document: Document to process (Mistral format dict, or file path, bytes, etc.) + document: Document to process - always a dict with type="document_url" or type="image_url" optional_params: Optional parameters for the request headers: Request headers diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 5acab8cbf2c..47cff8a2c0c 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -2,8 +2,14 @@ Main OCR function for LiteLLM. """ import asyncio +import base64 import contextvars +import mimetypes +import os +import re from functools import partial +from io import IOBase +from pathlib import Path from typing import Any, Coroutine, Dict, Optional, Union import httpx @@ -25,7 +31,7 @@ base_llm_http_handler = BaseLLMHTTPHandler() @client async def aocr( model: str, - document: Dict[str, str], + document: Dict[str, Any], api_key: Optional[str] = None, api_base: Optional[str] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, @@ -35,26 +41,27 @@ async def aocr( ) -> OCRResponse: """ Async OCR function. - + Args: model: Model name (e.g., "mistral/mistral-ocr-latest") document: Document to process in Mistral format: - {"type": "document_url", "document_url": "https://..."} for PDFs/docs or - {"type": "image_url", "image_url": "https://..."} for images + {"type": "document_url", "document_url": "https://..."} for PDFs/docs, + {"type": "image_url", "image_url": "https://..."} for images, or + {"type": "file", "file": } for local files api_key: Optional API key api_base: Optional API base URL timeout: Optional timeout custom_llm_provider: Optional custom LLM provider extra_headers: Optional extra headers **kwargs: Additional parameters (e.g., include_image_base64, pages, image_limit) - + Returns: OCRResponse in Mistral OCR format with pages, model, usage_info, etc. - + Example: ```python import litellm - + # OCR with PDF response = await litellm.aocr( model="mistral/mistral-ocr-latest", @@ -64,7 +71,7 @@ async def aocr( }, include_image_base64=True ) - + # OCR with image response = await litellm.aocr( model="mistral/mistral-ocr-latest", @@ -73,7 +80,7 @@ async def aocr( "image_url": "https://example.com/image.png" } ) - + # OCR with base64 encoded PDF response = await litellm.aocr( model="mistral/mistral-ocr-latest", @@ -82,6 +89,12 @@ async def aocr( "document_url": f"data:application/pdf;base64,{base64_pdf}" } ) + + # OCR with local file + response = await litellm.aocr( + model="mistral/mistral-ocr-latest", + document={"type": "file", "file": "/path/to/document.pdf"} + ) ``` """ local_vars = locals() @@ -135,7 +148,7 @@ async def aocr( @client def ocr( model: str, - document: Dict[str, str], + document: Dict[str, Any], api_key: Optional[str] = None, api_base: Optional[str] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, @@ -145,26 +158,27 @@ def ocr( ) -> Union[OCRResponse, Coroutine[Any, Any, OCRResponse]]: """ Synchronous OCR function. - + Args: model: Model name (e.g., "mistral/mistral-ocr-latest") document: Document to process in Mistral format: - {"type": "document_url", "document_url": "https://..."} for PDFs/docs or - {"type": "image_url", "image_url": "https://..."} for images + {"type": "document_url", "document_url": "https://..."} for PDFs/docs, + {"type": "image_url", "image_url": "https://..."} for images, or + {"type": "file", "file": } for local files api_key: Optional API key api_base: Optional API base URL timeout: Optional timeout custom_llm_provider: Optional custom LLM provider extra_headers: Optional extra headers **kwargs: Additional parameters (e.g., include_image_base64, pages, image_limit) - + Returns: OCRResponse in Mistral OCR format with pages, model, usage_info, etc. - + Example: ```python import litellm - + # OCR with PDF response = litellm.ocr( model="mistral/mistral-ocr-latest", @@ -174,7 +188,7 @@ def ocr( }, include_image_base64=True ) - + # OCR with image response = litellm.ocr( model="mistral/mistral-ocr-latest", @@ -183,7 +197,7 @@ def ocr( "image_url": "https://example.com/image.png" } ) - + # OCR with base64 encoded PDF response = litellm.ocr( model="mistral/mistral-ocr-latest", @@ -192,7 +206,13 @@ def ocr( "document_url": f"data:application/pdf;base64,{base64_pdf}" } ) - + + # OCR with local file + response = litellm.ocr( + model="mistral/mistral-ocr-latest", + document={"type": "file", "file": "/path/to/document.pdf"} + ) + # Access pages for page in response.pages: print(f"Page {page.index}: {page.markdown}") @@ -203,24 +223,38 @@ def ocr( litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) _is_async = kwargs.pop("aocr", False) is True - - # Validate document parameter format (Mistral spec) - if not isinstance(document, dict): - raise ValueError(f"document must be a dict with 'type' and URL field, got {type(document)}") - - doc_type = document.get("type") - if doc_type not in ["document_url", "image_url"]: - raise ValueError(f"Invalid document type: {doc_type}. Must be 'document_url' or 'image_url'") - model, custom_llm_provider, dynamic_api_key, dynamic_api_base = ( - litellm.get_llm_provider( - model=model, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - api_key=api_key, + # Validate document parameter format + if not isinstance(document, dict): + raise ValueError( + f"document must be a dict with 'type' and URL/file field, got {type(document)}" ) + + doc_type = document.get("type") + + # Handle file type: convert to document_url/image_url with base64 data URI + if doc_type == "file": + document = convert_file_document_to_url_document(document) + doc_type = document.get("type") + + if doc_type not in ["document_url", "image_url"]: + raise ValueError( + f"Invalid document type: {doc_type}. " + "Must be 'document_url', 'image_url', or 'file'" + ) + + ( + model, + custom_llm_provider, + dynamic_api_key, + dynamic_api_base, + ) = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, ) - + # Update with dynamic values if available if dynamic_api_key: api_key = dynamic_api_key @@ -228,11 +262,11 @@ def ocr( api_base = dynamic_api_base # Get provider config - ocr_provider_config: Optional[BaseOCRConfig] = ( - ProviderConfigManager.get_provider_ocr_config( - model=model, - provider=litellm.LlmProviders(custom_llm_provider), - ) + ocr_provider_config: Optional[ + BaseOCRConfig + ] = ProviderConfigManager.get_provider_ocr_config( + model=model, + provider=litellm.LlmProviders(custom_llm_provider), ) if ocr_provider_config is None: @@ -246,21 +280,21 @@ def ocr( # Get litellm params using GenericLiteLLMParams (same as responses API) litellm_params = GenericLiteLLMParams(**kwargs) - + # Extract OCR-specific parameters from kwargs supported_params = ocr_provider_config.get_supported_ocr_params(model=model) non_default_params = {} for param in supported_params: if param in kwargs: non_default_params[param] = kwargs.pop(param) - + # Map parameters to provider-specific format optional_params = ocr_provider_config.map_ocr_params( non_default_params=non_default_params, optional_params={}, model=model, ) - + verbose_logger.debug(f"OCR optional_params after mapping: {optional_params}") # Pre Call logging @@ -300,3 +334,111 @@ def ocr( extra_kwargs=kwargs, ) + +################################################# +# Public utilities — used by the SDK and the proxy +################################################# + +_MIME_PATTERN = re.compile(r"^[\w.+-]+/[\w.+-]+$") + +_MIME_TYPE_MAP = { + ".pdf": "application/pdf", + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".tiff": "image/tiff", + ".tif": "image/tiff", + ".bmp": "image/bmp", +} + + +def get_mime_type(file_path: str) -> str: + """ + Determine MIME type from file path extension. + + Falls back to mimetypes.guess_type, then to 'application/octet-stream'. + """ + ext = os.path.splitext(file_path)[1].lower() + mime = _MIME_TYPE_MAP.get(ext) + if mime: + return mime + guessed, _ = mimetypes.guess_type(file_path) + return guessed or "application/octet-stream" + + +def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str, str]: + """ + Convert a file-type document dict to a document_url-type document dict + with an inline base64 data URI. + + Accepts document dicts like: + {"type": "file", "file": "/path/to/document.pdf"} # file path string + {"type": "file", "file": Path("/path/to/doc.pdf")} # pathlib.Path + {"type": "file", "file": } # file-like object (BinaryIO) + {"type": "file", "file": b"raw bytes"} # raw bytes + + Returns: + {"type": "document_url", "document_url": "data:;base64,"} + or {"type": "image_url", "image_url": "data:;base64,"} + """ + file_input = document.get("file") + if file_input is None: + raise ValueError( + "document with type='file' must include a 'file' field containing " + "a file path (str), pathlib.Path, file-like object, or bytes" + ) + + file_bytes: bytes + mime_type: str = "application/octet-stream" + file_name: Optional[str] = None + + if isinstance(file_input, (str, Path)): + file_path = str(file_input) + if not os.path.isfile(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + mime_type = get_mime_type(file_path) + file_name = os.path.basename(file_path) + with open(file_path, "rb") as f: + file_bytes = f.read() + elif isinstance(file_input, bytes): + file_bytes = file_input + elif isinstance(file_input, IOBase) or hasattr(file_input, "read"): + if hasattr(file_input, "name"): + file_name = getattr(file_input, "name", None) + if file_name: + mime_type = get_mime_type(file_name) + file_bytes = file_input.read() + if isinstance(file_bytes, str): + file_bytes = file_bytes.encode("utf-8") + else: + raise ValueError( + f"Unsupported file input type: {type(file_input)}. " + "Expected str (file path), pathlib.Path, bytes, or a file-like object." + ) + + if not file_bytes: + raise ValueError("File is empty or could not be read") + + if "mime_type" in document: + mime_type = document["mime_type"] + + if not _MIME_PATTERN.match(mime_type): + raise ValueError(f"Invalid MIME type: {mime_type}") + + base64_data = base64.b64encode(file_bytes).decode("utf-8") + data_uri = f"data:{mime_type};base64,{base64_data}" + + if mime_type.startswith("image/"): + verbose_logger.debug( + f"OCR file input: Converted file to image_url data URI " + f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + ) + return {"type": "image_url", "image_url": data_uri} + else: + verbose_logger.debug( + f"OCR file input: Converted file to document_url data URI " + f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + ) + return {"type": "document_url", "document_url": data_uri} diff --git a/litellm/proxy/ocr_endpoints/endpoints.py b/litellm/proxy/ocr_endpoints/endpoints.py index c1092a06b48..4f31c762df1 100644 --- a/litellm/proxy/ocr_endpoints/endpoints.py +++ b/litellm/proxy/ocr_endpoints/endpoints.py @@ -1,9 +1,14 @@ #### OCR Endpoints ##### +import json +from typing import Any, Dict, Optional, cast + import orjson -from fastapi import APIRouter, Depends, Request, Response +from fastapi import APIRouter, Depends, Request, Response, UploadFile from fastapi.responses import ORJSONResponse +from litellm._logging import verbose_proxy_logger +from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -11,6 +16,171 @@ from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessin router = APIRouter() +def _build_document_from_upload( + file_content: bytes, + filename: Optional[str], + content_type: Optional[str], +) -> Dict[str, str]: + """ + Convert uploaded file bytes into a Mistral-format document dict with base64 data URI. + + Delegates to convert_file_document_to_url_document after resolving MIME type + from the upload's content_type header or filename. + """ + mime_type = content_type.split(";")[0].strip() if content_type else None + if not mime_type or mime_type == "application/octet-stream": + if filename: + mime_type = get_mime_type(filename) + + return convert_file_document_to_url_document( + { + "type": "file", + "file": file_content, + "mime_type": mime_type or "application/octet-stream", + } + ) + + +async def _parse_multipart_form(request: Request) -> Dict[str, Any]: + """ + Extract OCR data from a multipart form request. + + Uses the cached form if already parsed by auth middleware, + otherwise parses the form from the request. + + Returns: + A dict with 'document', 'model', and any other OCR params. + """ + try: + form = await request.form() + except Exception as e: + raise ValueError( + f"Failed to parse multipart form data: {str(e)}. " + "When using curl with --form/-F, do NOT set the Content-Type header " + "manually — curl will set it automatically with the required boundary." + ) + + uploaded_file = form.get("file") + # request.form() may return either a FastAPI or Starlette UploadFile + # depending on middleware; check both via isinstance (FastAPI's UploadFile + # is a subclass of Starlette's) and fall back to duck-type check. + if uploaded_file is None or ( + not isinstance(uploaded_file, UploadFile) and not hasattr(uploaded_file, "read") + ): + raise ValueError( + "Multipart OCR request must include a 'file' field with the document to process" + ) + + uploaded_file = cast(UploadFile, uploaded_file) + + # Seek to start in case the file was already partially read by middleware + await uploaded_file.seek(0) + file_content = await uploaded_file.read() + if not file_content: + raise ValueError("Uploaded file is empty") + + document = _build_document_from_upload( + file_content=file_content, + filename=uploaded_file.filename, + content_type=uploaded_file.content_type, + ) + + data: Dict[str, Any] = {"document": document} + + for field_name, field_value in form.items(): + if field_name in ("file", "document"): + continue + # Try to parse JSON values (e.g. pages=[0,1,2]) + if isinstance(field_value, str): + try: + data[field_name] = json.loads(field_value) + except (json.JSONDecodeError, ValueError): + data[field_name] = field_value + else: + data[field_name] = field_value + + verbose_proxy_logger.debug( + f"OCR multipart form request parsed - model: {data.get('model')}, " + f"document_type: {document['type']}, " + f"filename: {uploaded_file.filename}" + ) + + return data + + +async def _parse_ocr_request(request: Request) -> Dict[str, Any]: + """ + Parse an OCR request, supporting both JSON and multipart form data. + + JSON body (existing behavior): + { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "document_url", "document_url": "https://..."} + } + + Multipart form data (new): + - file: the uploaded file + - model: model name (form field) + - Any other OCR params as form fields (pages, include_image_base64, etc.) + + Returns: + A dict suitable for passing to the OCR processing pipeline. + """ + content_type = request.headers.get("content-type", "") + + if "multipart/form-data" in content_type.lower(): + return await _parse_multipart_form(request) + + # --- JSON body (existing behavior) --- + try: + body = await request.body() + except RuntimeError: + # Body stream was consumed by auth middleware (e.g., form parsing). + body = b"" + + if not body: + # The body may be empty because the auth middleware already parsed + # it as form data (e.g., _read_request_body called request.form()). + # Check if form data is available. + if getattr(request, "_form", None) is not None: + verbose_proxy_logger.debug( + "OCR request body is empty but form data is available from middleware — " + "processing as multipart form." + ) + return await _parse_multipart_form(request) + + raise ValueError( + "Empty request body. For file uploads, use multipart/form-data content type " + "with a file field. When using curl with --form/-F, do NOT set the Content-Type " + "header manually." + ) + + try: + data = orjson.loads(body) + except orjson.JSONDecodeError as e: + raise ValueError( + f"Invalid JSON in request body: {e}. " + "Ensure the request body is valid JSON with Content-Type: application/json, " + "or use multipart/form-data for file uploads." + ) + + # Security: reject type="file" documents received via JSON. + # The "file" document type is designed for local SDK usage where the + # caller and the process share a filesystem. In the proxy context the + # caller is remote, so allowing a file-path string would let an + # authenticated user read arbitrary files from the server's filesystem. + # File uploads must go through multipart/form-data instead. + doc = data.get("document") if isinstance(data, dict) else None + if isinstance(doc, dict) and doc.get("type") == "file": + raise ValueError( + "document type 'file' is not supported through the JSON API. " + "To upload a local file, use multipart/form-data with a 'file' field. " + "For JSON requests, use 'document_url' or 'image_url' document types." + ) + + return data + + @router.post( "/v1/ocr", dependencies=[Depends(user_api_key_auth)], @@ -30,23 +200,30 @@ async def ocr( ): """ OCR endpoint for extracting text from documents and images. - - Follows the Mistral OCR API spec: - https://docs.mistral.ai/capabilities/vision/#optical-character-recognition-ocr - - Example: + + Supports two input modes: + + **1. JSON body** (Mistral OCR API compatible): ```bash curl -X POST "http://localhost:4000/v1/ocr" \ -H "Authorization: Bearer sk-1234" \ -H "Content-Type: application/json" \ -d '{ - "model": "mistral/mistral-ocr-latest", + "model": "mistral-ocr", "document": { "type": "document_url", "document_url": "https://arxiv.org/pdf/2201.04234" } }' ``` + + **2. Multipart form file upload**: + ```bash + curl -X POST "http://localhost:4000/v1/ocr" \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@document.pdf" + ``` """ from litellm.proxy.proxy_server import ( general_settings, @@ -62,13 +239,14 @@ async def ocr( version, ) - # Read request body - body = await request.body() - data = orjson.loads(body) - - # Process request using ProxyBaseLLMRequestProcessing - processor = ProxyBaseLLMRequestProcessing(data=data) + data: dict = {} try: + # Parse request body (JSON or multipart form) + data = await _parse_ocr_request(request) + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + return await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, @@ -88,10 +266,10 @@ async def ocr( version=version, ) except Exception as e: + processor = ProxyBaseLLMRequestProcessing(data=data) raise await processor._handle_llm_api_exception( e=e, user_api_key_dict=user_api_key_dict, proxy_logging_obj=proxy_logging_obj, version=version, ) - diff --git a/tests/test_litellm/ocr/__init__.py b/tests/test_litellm/ocr/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/ocr/test_ocr_file_input.py b/tests/test_litellm/ocr/test_ocr_file_input.py new file mode 100644 index 00000000000..492253e2f11 --- /dev/null +++ b/tests/test_litellm/ocr/test_ocr_file_input.py @@ -0,0 +1,464 @@ +""" +Tests for OCR file input support. + +Tests that: +1. The SDK document parameter with type="file" correctly converts file paths, + file objects, and raw bytes to base64 data URIs before sending to providers. +2. The proxy _build_document_from_upload helper correctly handles uploaded file bytes. +3. The proxy rejects type="file" documents received via JSON (security guard). +4. The proxy returns user-friendly errors for invalid JSON bodies. +""" +import base64 +import os +import tempfile +from io import BytesIO +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import orjson +import pytest + +from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type + + +class TestGetMimeType: + def test_should_detect_pdf_mime_type(self): + assert get_mime_type("document.pdf") == "application/pdf" + + def test_should_detect_png_mime_type(self): + assert get_mime_type("image.png") == "image/png" + + def test_should_detect_jpg_mime_type(self): + assert get_mime_type("photo.jpg") == "image/jpeg" + + def test_should_detect_jpeg_mime_type(self): + assert get_mime_type("photo.jpeg") == "image/jpeg" + + def test_should_detect_gif_mime_type(self): + assert get_mime_type("animation.gif") == "image/gif" + + def test_should_detect_webp_mime_type(self): + assert get_mime_type("image.webp") == "image/webp" + + def test_should_detect_tiff_mime_type(self): + assert get_mime_type("scan.tiff") == "image/tiff" + + def test_should_detect_tif_mime_type(self): + assert get_mime_type("scan.tif") == "image/tiff" + + def test_should_detect_bmp_mime_type(self): + assert get_mime_type("bitmap.bmp") == "image/bmp" + + def test_should_be_case_insensitive(self): + assert get_mime_type("DOCUMENT.PDF") == "application/pdf" + assert get_mime_type("IMAGE.PNG") == "image/png" + + def test_should_fallback_for_unknown_extension(self): + result = get_mime_type("file.xyz123") + assert isinstance(result, str) + + +class TestConvertFileDocumentToUrlDocument: + def test_should_convert_pdf_file_path_to_document_url(self): + """File path to a PDF should produce type=document_url with base64 data URI.""" + pdf_content = b"%PDF-1.4 test content" + + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(pdf_content) + f.flush() + tmp_path = f.name + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == pdf_content + finally: + os.unlink(tmp_path) + + def test_should_convert_image_file_path_to_image_url(self): + """File path to a PNG image should produce type=image_url with base64 data URI.""" + png_content = b"\x89PNG\r\n\x1a\n fake png content" + + with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f: + f.write(png_content) + f.flush() + tmp_path = f.name + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + b64_data = result["image_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == png_content + finally: + os.unlink(tmp_path) + + def test_should_convert_pathlib_path(self): + """pathlib.Path objects should work the same as string paths.""" + content = b"test pdf content" + + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(content) + f.flush() + tmp_path = Path(f.name) + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + finally: + os.unlink(str(tmp_path)) + + def test_should_convert_raw_bytes(self): + """Raw bytes should be converted using a fallback MIME type.""" + content = b"raw bytes content" + + result = convert_file_document_to_url_document( + {"type": "file", "file": content} + ) + + assert result["type"] == "document_url" + assert "base64," in result["document_url"] + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == content + + def test_should_convert_raw_bytes_with_explicit_mime_type(self): + """Raw bytes with explicit mime_type should use the specified MIME type.""" + content = b"raw pdf content" + + result = convert_file_document_to_url_document( + {"type": "file", "file": content, "mime_type": "application/pdf"} + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + def test_should_convert_raw_bytes_with_image_mime_type(self): + """Raw bytes with an image MIME type should produce type=image_url.""" + content = b"raw image content" + + result = convert_file_document_to_url_document( + {"type": "file", "file": content, "mime_type": "image/jpeg"} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/jpeg;base64,") + + def test_should_convert_file_like_object(self): + """BytesIO and other file-like objects should be supported.""" + content = b"file-like content" + file_obj = BytesIO(content) + + result = convert_file_document_to_url_document( + {"type": "file", "file": file_obj} + ) + + assert result["type"] == "document_url" + assert "base64," in result["document_url"] + + def test_should_convert_file_like_object_with_name(self): + """File-like objects with a .name attribute should detect MIME from the name.""" + content = b"file-like png content" + file_obj = BytesIO(content) + file_obj.name = "test_image.png" + + result = convert_file_document_to_url_document( + {"type": "file", "file": file_obj} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + def test_should_raise_error_for_missing_file_field(self): + """Missing 'file' field should raise ValueError.""" + with pytest.raises(ValueError, match="must include a 'file' field"): + convert_file_document_to_url_document({"type": "file"}) + + def test_should_raise_error_for_nonexistent_file_path(self): + """Non-existent file path should raise FileNotFoundError.""" + with pytest.raises(FileNotFoundError, match="File not found"): + convert_file_document_to_url_document( + {"type": "file", "file": "/nonexistent/path/to/file.pdf"} + ) + + def test_should_raise_error_for_empty_file(self): + """Empty file should raise ValueError.""" + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + tmp_path = f.name + + try: + with pytest.raises(ValueError, match="File is empty"): + convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + finally: + os.unlink(tmp_path) + + def test_should_raise_error_for_unsupported_type(self): + """Unsupported file input types should raise ValueError.""" + with pytest.raises(ValueError, match="Unsupported file input type"): + convert_file_document_to_url_document({"type": "file", "file": 12345}) + + def test_should_raise_error_for_invalid_mime_type(self): + """MIME types with special characters should be rejected.""" + content = b"some content" + with pytest.raises(ValueError, match="Invalid MIME type"): + convert_file_document_to_url_document( + {"type": "file", "file": content, "mime_type": "text/html; charset=utf-8\nX-Injected: true"} + ) + + def test_should_override_mime_type_for_file_path(self): + """Explicit mime_type should override auto-detection from extension.""" + content = b"some content" + + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(content) + f.flush() + tmp_path = f.name + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path, "mime_type": "image/png"} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + finally: + os.unlink(tmp_path) + + +class TestBuildDocumentFromUpload: + """Test the proxy endpoint's file upload to document conversion helper.""" + + @pytest.fixture(autouse=True) + def _import_helper(self): + """Import the proxy helper, skip if proxy deps aren't installed.""" + try: + from litellm.proxy.ocr_endpoints.endpoints import ( + _build_document_from_upload, + ) + + self._build = _build_document_from_upload + except ImportError: + pytest.skip("Proxy dependencies (fastapi/orjson) not installed") + + def test_should_build_document_url_for_pdf(self): + content = b"%PDF-1.4 test content" + + result = self._build( + file_content=content, + filename="document.pdf", + content_type="application/pdf", + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == content + + def test_should_build_image_url_for_png(self): + content = b"\x89PNG fake png" + + result = self._build( + file_content=content, + filename="screenshot.png", + content_type="image/png", + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + def test_should_build_image_url_for_jpeg(self): + content = b"\xff\xd8\xff fake jpeg" + + result = self._build( + file_content=content, + filename="photo.jpg", + content_type="image/jpeg", + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/jpeg;base64,") + + def test_should_detect_mime_from_filename_when_content_type_is_octet_stream(self): + content = b"pdf content" + + result = self._build( + file_content=content, + filename="report.pdf", + content_type="application/octet-stream", + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + def test_should_detect_mime_from_filename_when_content_type_is_none(self): + content = b"png content" + + result = self._build( + file_content=content, + filename="image.png", + content_type=None, + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + def test_should_fallback_to_octet_stream_for_unknown(self): + content = b"unknown content" + + result = self._build( + file_content=content, + filename=None, + content_type=None, + ) + + assert result["type"] == "document_url" + assert "application/octet-stream" in result["document_url"] + + def test_should_preserve_base64_content_correctly(self): + content = b"Hello, World! \x00\x01\x02\xff" + + result = self._build( + file_content=content, + filename="test.pdf", + content_type="application/pdf", + ) + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == content + + def test_should_strip_mime_parameters_from_content_type(self): + """Content-Type with parameters (e.g. charset) should be stripped to the base MIME type.""" + content = b"%PDF-1.4 test" + + result = self._build( + file_content=content, + filename="doc.pdf", + content_type="application/pdf; charset=utf-8", + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + def test_should_strip_mime_parameters_with_multiple_params(self): + """Content-Type with multiple parameters should still be stripped correctly.""" + content = b"image data" + + result = self._build( + file_content=content, + filename="img.png", + content_type="image/png; charset=utf-8; boundary=something", + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + +class TestProxySecurityGuard: + """Test that the proxy rejects type='file' documents in JSON requests + and that multipart form fields cannot override the constructed document.""" + + @pytest.fixture(autouse=True) + def _import_helpers(self): + """Import the proxy helpers, skip if proxy deps aren't installed.""" + try: + from litellm.proxy.ocr_endpoints.endpoints import ( + _parse_multipart_form, + _parse_ocr_request, + ) + + self._parse = _parse_ocr_request + self._parse_multipart = _parse_multipart_form + except ImportError: + pytest.skip("Proxy dependencies (fastapi/orjson) not installed") + + @pytest.mark.asyncio + async def test_should_reject_file_type_document_in_json_body(self): + """type='file' in a JSON body must be rejected to prevent server-side file reads.""" + body = orjson.dumps( + { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "file", "file": "/etc/passwd"}, + } + ) + + mock_request = MagicMock() + mock_request.headers = {"content-type": "application/json"} + mock_request.body = AsyncMock(return_value=body) + mock_request._form = None + + with pytest.raises(ValueError, match="not supported through the JSON API"): + await self._parse(mock_request) + + @pytest.mark.asyncio + async def test_should_accept_document_url_type_in_json_body(self): + """type='document_url' in a JSON body should pass through normally.""" + expected = { + "model": "mistral/mistral-ocr-latest", + "document": { + "type": "document_url", + "document_url": "https://example.com/doc.pdf", + }, + } + body = orjson.dumps(expected) + + mock_request = MagicMock() + mock_request.headers = {"content-type": "application/json"} + mock_request.body = AsyncMock(return_value=body) + mock_request._form = None + + result = await self._parse(mock_request) + assert result["document"]["type"] == "document_url" + + @pytest.mark.asyncio + async def test_should_raise_on_invalid_json_body(self): + """Invalid JSON should produce a user-friendly ValueError.""" + mock_request = MagicMock() + mock_request.headers = {"content-type": "application/json"} + mock_request.body = AsyncMock(return_value=b"not valid json{{{") + mock_request._form = None + + with pytest.raises(ValueError, match="Invalid JSON in request body"): + await self._parse(mock_request) + + @pytest.mark.asyncio + async def test_should_ignore_document_form_field_injection(self): + """A 'document' form field must not override the document built from the uploaded file.""" + from starlette.datastructures import UploadFile + + file_content = b"%PDF-1.4 legit content" + upload = UploadFile(filename="legit.pdf", file=BytesIO(file_content)) + + injected = '{"type": "file", "file": "/etc/passwd"}' + + mock_form = { + "file": upload, + "model": "mistral/mistral-ocr-latest", + "document": injected, + } + + mock_request = MagicMock() + mock_request.headers = {"content-type": "multipart/form-data; boundary=---"} + mock_request.form = AsyncMock(return_value=mock_form) + + result = await self._parse_multipart(mock_request) + + assert result["document"]["type"] == "document_url" + assert result["document"]["document_url"].startswith("data:application/pdf;base64,") + assert result["model"] == "mistral/mistral-ocr-latest" From 1144d05cbab94c473964ab8be575015f767cd073 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 15:57:56 -0300 Subject: [PATCH 30/46] feat(models): add gpt-audio-1.5 to model cost map New OpenAI audio model released 2026-02-23. Adds pricing and capability metadata for gpt-audio-1.5 (128K context, 16K output, audio I/O). Closes #22269 --- model_prices_and_context_window.json | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e4d7a6a02f2..e9002084de9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -19085,6 +19085,42 @@ "supports_tool_choice": true, "supports_vision": false }, + "gpt-audio-1.5": { + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/realtime", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "gpt-audio-2025-08-28": { "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, From 29bb73ffcace33e65f39429493a10b1ecb76c909 Mon Sep 17 00:00:00 2001 From: Gaurav Singh <103016722+gavksingh@users.noreply.github.com> Date: Fri, 27 Feb 2026 13:59:08 -0500 Subject: [PATCH 31/46] fix(mcp): strip stale mcp-session-id header to prevent 400 in multi-worker deployments (#20992) (#21417) In a multi-worker Uvicorn setup, a client that reconnects to a different worker sends an mcp-session-id that the new worker has never seen. The MCP SDK returns 400 because the session is unknown. Fix: add _handle_stale_mcp_session() which inspects the inbound mcp-session-id header before the request reaches the SDK. If the session is not in this worker's _server_instances: - Non-DELETE: strip the header so the SDK creates a fresh session - DELETE: return 200 immediately (idempotent, session already gone) No new dependencies, no Redis, no latency added to the hot path. Fixes https://github.com/BerriAI/litellm/issues/20992 --- .../proxy/_experimental/mcp_server/server.py | 99 ++++++++++++------- tests/mcp_tests/test_mcp_server.py | 3 +- 2 files changed, 64 insertions(+), 38 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 48c837c1e4f..5b3d5bd60e2 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -5,6 +5,7 @@ LiteLLM MCP Server Routes import asyncio import contextlib + import traceback import uuid from datetime import datetime @@ -84,6 +85,7 @@ except ImportError as e: _SESSION_MANAGERS_INITIALIZED = False _INITIALIZATION_LOCK = asyncio.Lock() + if MCP_AVAILABLE: from mcp.server import Server @@ -1919,65 +1921,86 @@ if MCP_AVAILABLE: mgr: "StreamableHTTPSessionManager", ) -> bool: """ - Handle stale MCP session IDs to prevent "Session not found" errors. - - When clients reconnect after a server restart or session cleanup, they may - send a session ID that no longer exists. This function handles two scenarios: - - 1. Non-DELETE requests: Strip the stale session ID header so the session - manager creates a fresh session transparently. - - 2. DELETE requests: Return success (200) immediately for idempotent behavior, - since the desired state (session doesn't exist) is already achieved. + Inspect the incoming ``mcp-session-id`` header **before** the + request reaches the MCP SDK. If the session is stale (not known + to this worker), strip the header so the SDK creates a fresh + stateless session instead of returning a 400. Returns: - True if the request was handled (DELETE on non-existent session) - False if the request should continue to the session manager + True if the request was fully handled (e.g. DELETE on + non-existent session). False if the request should continue + to the session manager. - Fixes https://github.com/BerriAI/litellm/issues/20292 + Fixes https://github.com/BerriAI/litellm/issues/20992 """ _mcp_session_header = b"mcp-session-id" + _headers = scope.get("headers", []) + + def _normalize_header_name(header_name: Any) -> Optional[bytes]: + if isinstance(header_name, bytes): + return header_name.lower() + if isinstance(header_name, str): + return header_name.lower().encode("utf-8", errors="replace") + return None + _session_id: Optional[str] = None - for header_name, header_value in scope.get("headers", []): - if header_name == _mcp_session_header: - _session_id = header_value.decode("utf-8", errors="replace") + for header_name, header_value in _headers: + if _normalize_header_name(header_name) == _mcp_session_header: + if isinstance(header_value, bytes): + _session_id = header_value.decode("utf-8", errors="replace") + else: + _session_id = str(header_value) break if _session_id is None: return False + # Check in-memory session tracking known_sessions = getattr(mgr, "_server_instances", None) - if known_sessions is None or _session_id in known_sessions: - # Session exists or we can't check - let the session manager handle it + # If we cannot inspect known_sessions, let the manager handle it + if known_sessions is None: return False - # Session doesn't exist - handle based on request method + # If session exists in this worker's memory, let the manager handle it + try: + if _session_id in known_sessions: + return False + except Exception: + verbose_logger.debug( + "Unable to inspect active MCP sessions for '%s'. " + "Deferring to session manager.", + _session_id, + ) + return False + + # --- Session not in this worker's memory --- method = scope.get("method", "").upper() - + if method == "DELETE": - # Idempotent DELETE: session doesn't exist, return success verbose_logger.info( - f"DELETE request for non-existent MCP session '{_session_id}'. " - "Returning success (idempotent DELETE)." + "DELETE request for non-existent MCP session '%s'. " + "Returning success (idempotent DELETE).", + _session_id, ) success_response = JSONResponse( status_code=200, - content={"message": "Session terminated successfully"} + content={"message": "Session terminated successfully"}, ) await success_response(scope, receive, send) return True - else: - # Non-DELETE: strip stale session ID to allow new session creation - verbose_logger.warning( - "MCP session ID '%s' not found in active sessions. " - "Stripping stale header to force new session creation.", - _session_id, - ) - scope["headers"] = [ - (k, v) for k, v in scope["headers"] - if k != _mcp_session_header - ] - return False + + # Non-DELETE: strip stale session ID to allow new session creation + verbose_logger.warning( + "MCP session ID '%s' not found in this worker's memory. " + "Stripping stale header to force new session creation.", + _session_id, + ) + scope["headers"] = [ + (k, v) + for k, v in _headers + if _normalize_header_name(k) != _mcp_session_header + ] + return False async def handle_streamable_http_mcp( scope: Scope, receive: Receive, send: Send @@ -2055,7 +2078,9 @@ if MCP_AVAILABLE: # Handle stale session IDs - either strip them for reconnection # or return success for idempotent DELETE operations - handled = await _handle_stale_mcp_session(scope, receive, send, session_manager) + handled = await _handle_stale_mcp_session( + scope, receive, send, session_manager + ) if handled: # Request was fully handled (e.g., DELETE on non-existent session) return diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 930b0a03042..dc1e2068365 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -427,7 +427,8 @@ async def test_streamable_http_mcp_handler_mock(): # Call the handler await handle_streamable_http_mcp(mock_scope, mock_receive, mock_send) - # Verify session manager handle_request was called + # Verify session manager handle_request was called with correct args + # send is passed directly (no wrapper) mock_session_manager.handle_request.assert_called_once_with( mock_scope, mock_receive, mock_send ) From 1f887547f6652d7bc5f87fa44f8563131b853124 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 16:01:09 -0300 Subject: [PATCH 32/46] feat(models): add gpt-realtime-1.5 to model cost map New OpenAI realtime model released 2026-02-23. Adds pricing and capability metadata for gpt-realtime-1.5 (32K context, 4K output, audio/image/text I/O). Unlike gpt-realtime, this model also supports Chat Completions and Responses endpoints (not just WebSocket). Closes #22266 --- model_prices_and_context_window.json | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e4d7a6a02f2..041f76af70b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20802,6 +20802,40 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "gpt-realtime-1.5": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1.6e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, From c7ab631bb045caebc588b94a890305612d47146e Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 16:06:10 -0300 Subject: [PATCH 33/46] fix(audio): detect gpt-realtime models as audio-capable for Chat Completions gpt-realtime-1.5 supports Chat Completions with audio params but is_model_gpt_audio_model only checked for "audio" in the model name. Add "realtime" check so the audio parameter is passed through correctly. --- litellm/llms/openai/chat/gpt_audio_transformation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/chat/gpt_audio_transformation.py b/litellm/llms/openai/chat/gpt_audio_transformation.py index 581ffea2db4..66270b1da9d 100644 --- a/litellm/llms/openai/chat/gpt_audio_transformation.py +++ b/litellm/llms/openai/chat/gpt_audio_transformation.py @@ -29,7 +29,9 @@ class OpenAIGPTAudioConfig(OpenAIGPTConfig): return all_openai_params + audio_specific_params def is_model_gpt_audio_model(self, model: str) -> bool: - if model in litellm.open_ai_chat_completion_models and "audio" in model: + if model in litellm.open_ai_chat_completion_models and ( + "audio" in model or "realtime" in model + ): return True return False From da73e54b1bd9e8c8974e912500b989ba080f1c02 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 16:10:01 -0300 Subject: [PATCH 34/46] fix: gpt-realtime-1.5 only supports /v1/realtime endpoint Remove /v1/chat/completions and /v1/responses from supported_endpoints and revert the audio model detection change since gpt-realtime-1.5 does not go through Chat Completions. --- litellm/llms/openai/chat/gpt_audio_transformation.py | 4 +--- model_prices_and_context_window.json | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_audio_transformation.py b/litellm/llms/openai/chat/gpt_audio_transformation.py index 66270b1da9d..581ffea2db4 100644 --- a/litellm/llms/openai/chat/gpt_audio_transformation.py +++ b/litellm/llms/openai/chat/gpt_audio_transformation.py @@ -29,9 +29,7 @@ class OpenAIGPTAudioConfig(OpenAIGPTConfig): return all_openai_params + audio_specific_params def is_model_gpt_audio_model(self, model: str) -> bool: - if model in litellm.open_ai_chat_completion_models and ( - "audio" in model or "realtime" in model - ): + if model in litellm.open_ai_chat_completion_models and "audio" in model: return True return False diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 041f76af70b..66d5f8b7c35 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20816,8 +20816,6 @@ "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses", "/v1/realtime" ], "supported_modalities": [ From 1ca4dd8542bfcd3fc2ec691b6125d78975c546e2 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 16:11:15 -0300 Subject: [PATCH 35/46] fix: gpt-audio-1.5 only supports /v1/chat/completions endpoint --- model_prices_and_context_window.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e9002084de9..35b820c1a6e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -19096,10 +19096,7 @@ "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses", - "/v1/realtime", - "/v1/batch" + "/v1/chat/completions" ], "supported_modalities": [ "text", From ad9c70ec5d14e8947a33d796b9c0b85ae32b1b16 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Fri, 27 Feb 2026 11:14:13 -0800 Subject: [PATCH 36/46] Add LLMClientCache regression tests for httpx client eviction safety MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regression tests for PR #22247 — ensures cache eviction (capacity and TTL) does not close httpx clients that are still in use. --- .../caching/test_redis_connection_pool.py | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/tests/test_litellm/caching/test_redis_connection_pool.py b/tests/test_litellm/caching/test_redis_connection_pool.py index f6e429ceff9..3d808438850 100644 --- a/tests/test_litellm/caching/test_redis_connection_pool.py +++ b/tests/test_litellm/caching/test_redis_connection_pool.py @@ -1,15 +1,13 @@ -""" -Regression tests for Redis connection pool leak fixes (RC1-RC5). - -Tests are pure unit tests — no Redis server required. -""" +"""Redis connection pool and LLMClientCache eviction tests.""" from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import redis.asyncio as async_redis from litellm._redis import get_redis_async_client, get_redis_connection_pool +from litellm.caching.llm_caching_handler import LLMClientCache def test_url_config_uses_passed_pool(): @@ -129,3 +127,33 @@ async def test_disconnect_idempotent(): await cache.disconnect() # should not raise +# Regression: cache eviction must not close shared httpx clients (PR #22247) + +@pytest.mark.asyncio +async def test_httpx_client_survives_capacity_eviction(): + """Evicting an httpx client from LLMClientCache must NOT close it.""" + cache = LLMClientCache(max_size_in_memory=1, default_ttl=600) + client = httpx.AsyncClient() + + cache.set_cache("client_1", client) + # Exceed capacity — client_1 gets evicted + cache.set_cache("client_2", "other") + + assert not client.is_closed + await client.aclose() + + +@pytest.mark.asyncio +async def test_httpx_client_survives_ttl_eviction(): + """Evicting an httpx client via TTL expiry must NOT close it.""" + cache = LLMClientCache(max_size_in_memory=200, default_ttl=600) + client = httpx.AsyncClient() + + # TTL=0 so it expires immediately + cache.set_cache("client_1", client, ttl=0) + cache.evict_cache() + + assert not client.is_closed + await client.aclose() + + From 6490ad1d484d848ede12e6f390922893ed4d2466 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Fri, 27 Feb 2026 12:43:03 -0800 Subject: [PATCH 37/46] Revert "Add LLMClientCache regression tests for httpx client eviction safety" This reverts commit ad9c70ec5d14e8947a33d796b9c0b85ae32b1b16. --- .../caching/test_redis_connection_pool.py | 38 +++---------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/tests/test_litellm/caching/test_redis_connection_pool.py b/tests/test_litellm/caching/test_redis_connection_pool.py index 3d808438850..f6e429ceff9 100644 --- a/tests/test_litellm/caching/test_redis_connection_pool.py +++ b/tests/test_litellm/caching/test_redis_connection_pool.py @@ -1,13 +1,15 @@ -"""Redis connection pool and LLMClientCache eviction tests.""" +""" +Regression tests for Redis connection pool leak fixes (RC1-RC5). + +Tests are pure unit tests — no Redis server required. +""" from unittest.mock import AsyncMock, MagicMock, patch -import httpx import pytest import redis.asyncio as async_redis from litellm._redis import get_redis_async_client, get_redis_connection_pool -from litellm.caching.llm_caching_handler import LLMClientCache def test_url_config_uses_passed_pool(): @@ -127,33 +129,3 @@ async def test_disconnect_idempotent(): await cache.disconnect() # should not raise -# Regression: cache eviction must not close shared httpx clients (PR #22247) - -@pytest.mark.asyncio -async def test_httpx_client_survives_capacity_eviction(): - """Evicting an httpx client from LLMClientCache must NOT close it.""" - cache = LLMClientCache(max_size_in_memory=1, default_ttl=600) - client = httpx.AsyncClient() - - cache.set_cache("client_1", client) - # Exceed capacity — client_1 gets evicted - cache.set_cache("client_2", "other") - - assert not client.is_closed - await client.aclose() - - -@pytest.mark.asyncio -async def test_httpx_client_survives_ttl_eviction(): - """Evicting an httpx client via TTL expiry must NOT close it.""" - cache = LLMClientCache(max_size_in_memory=200, default_ttl=600) - client = httpx.AsyncClient() - - # TTL=0 so it expires immediately - cache.set_cache("client_1", client, ttl=0) - cache.evict_cache() - - assert not client.is_closed - await client.aclose() - - From 0b7e9a19719118db15226489bda940671ac18441 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Fri, 27 Feb 2026 12:43:08 -0800 Subject: [PATCH 38/46] Add e2e tests: httpx clients survive LLMClientCache eviction Tests go through the real get_async_httpx_client() code path to verify clients remain usable after both capacity eviction and TTL expiry. Regression tests for PR #22247. --- .../caching/test_llm_client_cache_e2e.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/test_litellm/caching/test_llm_client_cache_e2e.py diff --git a/tests/test_litellm/caching/test_llm_client_cache_e2e.py b/tests/test_litellm/caching/test_llm_client_cache_e2e.py new file mode 100644 index 00000000000..b1f8be3b484 --- /dev/null +++ b/tests/test_litellm/caching/test_llm_client_cache_e2e.py @@ -0,0 +1,44 @@ +"""e2e tests: httpx clients obtained via get_async_httpx_client must remain +usable after LLMClientCache evicts their cache entry.""" + +import pytest + +import litellm +from litellm.caching.llm_caching_handler import LLMClientCache +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + + +@pytest.fixture(autouse=True) +def _tiny_client_cache(monkeypatch): + """Replace the global client cache with a size-1 cache so eviction + triggers on the second insert.""" + cache = LLMClientCache(max_size_in_memory=1, default_ttl=600) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", cache) + yield cache + + +@pytest.mark.asyncio +async def test_evicted_client_is_not_closed(): + """Get a client via get_async_httpx_client, evict it by caching a second + one, then verify the first client's transport is still open.""" + client_a = get_async_httpx_client(llm_provider="provider_a") + # This evicts client_a from cache (capacity=1) + get_async_httpx_client(llm_provider="provider_b") + + assert not client_a.client.is_closed + + +@pytest.mark.asyncio +async def test_expired_client_is_not_closed(): + """Get a client, expire it via TTL, then verify the client is still open.""" + cache = litellm.in_memory_llm_clients_cache + client = get_async_httpx_client(llm_provider="provider_ttl") + + # Force the entry to expire and trigger eviction + for key in list(cache.ttl_dict.keys()): + cache.ttl_dict[key] = 0 + # Also fix the heap entry so evict_cache finds it + cache.expiration_heap = [(0, key) for _, key in cache.expiration_heap] + cache.evict_cache() + + assert not client.client.is_closed From dce597b806ea029d8a15390c2dde09fa83a463da Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Fri, 27 Feb 2026 12:58:33 -0800 Subject: [PATCH 39/46] Close httpx clients after assertions to prevent resource leaks --- tests/test_litellm/caching/test_llm_client_cache_e2e.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/caching/test_llm_client_cache_e2e.py b/tests/test_litellm/caching/test_llm_client_cache_e2e.py index b1f8be3b484..a7d012d2269 100644 --- a/tests/test_litellm/caching/test_llm_client_cache_e2e.py +++ b/tests/test_litellm/caching/test_llm_client_cache_e2e.py @@ -23,9 +23,11 @@ async def test_evicted_client_is_not_closed(): one, then verify the first client's transport is still open.""" client_a = get_async_httpx_client(llm_provider="provider_a") # This evicts client_a from cache (capacity=1) - get_async_httpx_client(llm_provider="provider_b") + client_b = get_async_httpx_client(llm_provider="provider_b") assert not client_a.client.is_closed + await client_a.client.aclose() + await client_b.client.aclose() @pytest.mark.asyncio @@ -42,3 +44,4 @@ async def test_expired_client_is_not_closed(): cache.evict_cache() assert not client.client.is_closed + await client.client.aclose() From 2613277ede94529de7b2542f579fe02486cd48e8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 27 Feb 2026 13:31:17 -0800 Subject: [PATCH 40/46] [Feature] UI - Projects: Add Projects page with list and create flows Add a new Projects page under ACCESS CONTROL in the left nav, positioned between Teams and Internal Users. Projects sit between teams and keys in the hierarchy for use-case management. - ProjectsPage with Ant Design table (list, search, pagination) - CreateProjectModal with team-scoped model selection - React-query hooks for /project/list and /project/new - New badge on Projects, removed from Access Groups - Page metadata entry for test compliance Co-Authored-By: Claude Opus 4.6 (1M context) --- .../hooks/projects/useCreateProject.ts | 70 ++++ .../(dashboard)/hooks/projects/useProjects.ts | 87 +++++ ui/litellm-dashboard/src/app/page.tsx | 3 + .../ProjectModals/CreateProjectModal.tsx | 348 ++++++++++++++++++ .../src/components/Projects/ProjectsPage.tsx | 211 +++++++++++ .../src/components/Projects/types.ts | 14 + .../src/components/leftnav.tsx | 30 +- .../src/components/page_metadata.ts | 1 + 8 files changed, 753 insertions(+), 11 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useProjects.ts create mode 100644 ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx create mode 100644 ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx create mode 100644 ui/litellm-dashboard/src/components/Projects/types.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts new file mode 100644 index 00000000000..3943f23794e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts @@ -0,0 +1,70 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { + getProxyBaseUrl, + getGlobalLitellmHeaderName, + deriveErrorMessage, + handleError, +} from "@/components/networking"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { ProjectResponse, projectKeys } from "./useProjects"; + +// ── Types ──────────────────────────────────────────────────────────────────── + +export interface ProjectCreateParams { + project_alias?: string; + description?: string; + team_id: string; + models?: string[]; + max_budget?: number; + blocked?: boolean; + metadata?: Record; + model_rpm_limit?: Record; + model_tpm_limit?: Record; +} + +// ── Fetch function ─────────────────────────────────────────────────────────── + +const createProject = async ( + accessToken: string, + params: ProjectCreateParams, +): Promise => { + const baseUrl = getProxyBaseUrl(); + const url = `${baseUrl}/project/new`; + + const response = await fetch(url, { + method: "POST", + headers: { + [getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(params), + }); + + if (!response.ok) { + const errorData = await response.json(); + const errorMessage = deriveErrorMessage(errorData); + handleError(errorMessage); + throw new Error(errorMessage); + } + + return response.json(); +}; + +// ── Hook ───────────────────────────────────────────────────────────────────── + +export const useCreateProject = () => { + const { accessToken } = useAuthorized(); + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: async (params) => { + if (!accessToken) { + throw new Error("Access token is required"); + } + return createProject(accessToken, params); + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: projectKeys.all }); + }, + }); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useProjects.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useProjects.ts new file mode 100644 index 00000000000..85c8b25645c --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useProjects.ts @@ -0,0 +1,87 @@ +import { useQuery } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; +import { + getProxyBaseUrl, + getGlobalLitellmHeaderName, + deriveErrorMessage, + handleError, +} from "@/components/networking"; +import { all_admin_roles } from "@/utils/roles"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; + +// ── Types ──────────────────────────────────────────────────────────────────── + +export interface ProjectBudget { + budget_id: string; + max_budget: number | null; + soft_budget: number | null; + max_parallel_requests: number | null; + tpm_limit: number | null; + rpm_limit: number | null; + model_max_budget: Record | null; + budget_duration: string | null; +} + +export interface ProjectResponse { + project_id: string; + project_alias: string | null; + description: string | null; + team_id: string | null; + budget_id: string | null; + metadata: Record | null; + models: string[]; + spend: number; + model_spend: Record | null; + model_rpm_limit: Record | null; + model_tpm_limit: Record | null; + blocked: boolean; + object_permission_id: string | null; + created_at: string; + created_by: string; + updated_at: string; + updated_by: string; + litellm_budget_table: ProjectBudget | null; +} + +// ── Query keys (shared across project hooks) ───────────────────────────────── + +export const projectKeys = createQueryKeys("projects"); + +// ── Fetch function ─────────────────────────────────────────────────────────── + +const fetchProjects = async ( + accessToken: string, +): Promise => { + const baseUrl = getProxyBaseUrl(); + const url = `${baseUrl}/project/list`; + + const response = await fetch(url, { + method: "GET", + headers: { + [getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.json(); + const errorMessage = deriveErrorMessage(errorData); + handleError(errorMessage); + throw new Error(errorMessage); + } + + return response.json(); +}; + +// ── Hook ───────────────────────────────────────────────────────────────────── + +export const useProjects = () => { + const { accessToken, userRole } = useAuthorized(); + + return useQuery({ + queryKey: projectKeys.list({}), + queryFn: async () => fetchProjects(accessToken!), + enabled: + Boolean(accessToken) && all_admin_roles.includes(userRole || ""), + }); +}; diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 258c2ccb0e0..0b2f467e8f8 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -37,6 +37,7 @@ import UIThemeSettings from "@/components/ui_theme_settings"; import Usage from "@/components/usage"; import UserDashboard from "@/components/user_dashboard"; import { AccessGroupsPage } from "@/components/AccessGroups/AccessGroupsPage"; +import { ProjectsPage } from "@/components/Projects/ProjectsPage"; import VectorStoreManagement from "@/components/vector_store_management"; import ToolPolicies from "@/components/ToolPolicies"; import SpendLogsTable from "@/components/view_logs"; @@ -547,6 +548,8 @@ function CreateKeyPageContent() { ) : page == "access-groups" ? ( + ) : page == "projects" ? ( + ) : page == "vector-stores" ? ( ) : page == "tool-policies" ? ( diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx new file mode 100644 index 00000000000..14b4d70b743 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx @@ -0,0 +1,348 @@ +import { useEffect, useState } from "react"; +import { + Alert, + Modal, + Form, + Input, + Select, + Switch, + InputNumber, + Collapse, + Button, + Col, + Flex, + Row, + Space, + Divider, + Typography, + message, +} from "antd"; +import { FolderAddOutlined, PlusOutlined, MinusCircleOutlined } from "@ant-design/icons"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; +import { useCreateProject, ProjectCreateParams } from "@/app/(dashboard)/hooks/projects/useCreateProject"; +import { Team } from "../../key_team_helpers/key_list"; +import { fetchTeamModels } from "../../organisms/create_key_button"; +import { getModelDisplayName } from "../../key_team_helpers/fetch_available_models_team_key"; + +interface CreateProjectModalProps { + isOpen: boolean; + onClose: () => void; +} + +export function CreateProjectModal({ isOpen, onClose }: CreateProjectModalProps) { + const [form] = Form.useForm(); + const { accessToken, userId, userRole } = useAuthorized(); + const { data: teams } = useTeams(); + const createMutation = useCreateProject(); + + const [selectedTeam, setSelectedTeam] = useState(null); + const [modelsToPick, setModelsToPick] = useState([]); + + // Fetch team-scoped models when team selection changes + useEffect(() => { + if (userId && userRole && accessToken && selectedTeam) { + fetchTeamModels(userId, userRole, accessToken, selectedTeam.team_id).then((models) => { + const allModels = Array.from(new Set([...(selectedTeam.models ?? []), ...models])); + setModelsToPick(allModels); + }); + } else { + setModelsToPick([]); + } + form.setFieldValue("models", []); + }, [selectedTeam, accessToken, userId, userRole, form]); + + const handleSubmit = async () => { + try { + const values = await form.validateFields(); + + // Build model-specific limits from the dynamic form list + const modelRpmLimit: Record = {}; + const modelTpmLimit: Record = {}; + for (const entry of values.modelLimits ?? []) { + if (entry.model) { + if (entry.rpm != null) modelRpmLimit[entry.model] = entry.rpm; + if (entry.tpm != null) modelTpmLimit[entry.model] = entry.tpm; + } + } + + // Build metadata from the dynamic form list + const metadata: Record = {}; + for (const entry of values.metadata ?? []) { + if (entry.key) metadata[entry.key] = entry.value; + } + + const params: ProjectCreateParams = { + project_alias: values.project_alias, + description: values.description, + team_id: values.team_id, + models: values.models ?? [], + max_budget: values.max_budget, + blocked: values.isBlocked ?? false, + ...(Object.keys(modelRpmLimit).length > 0 && { model_rpm_limit: modelRpmLimit }), + ...(Object.keys(modelTpmLimit).length > 0 && { model_tpm_limit: modelTpmLimit }), + ...(Object.keys(metadata).length > 0 && { metadata }), + }; + + createMutation.mutate(params, { + onSuccess: () => { + message.success("Project created successfully"); + form.resetFields(); + setSelectedTeam(null); + setModelsToPick([]); + onClose(); + }, + onError: (error) => { + message.error(error.message || "Failed to create project"); + }, + }); + } catch (error) { + console.error("Validation failed:", error); + } + }; + + const handleCancel = () => { + form.resetFields(); + setSelectedTeam(null); + setModelsToPick([]); + onClose(); + }; + + const handleTeamChange = (teamId: string) => { + const team = teams?.find((t) => t.team_id === teamId) ?? null; + setSelectedTeam(team); + }; + + return ( + + Create New Project + + } + open={isOpen} + onCancel={handleCancel} + width={720} + footer={[ + , + , + ]} + > +
+ {/* Basic Info */} + + Basic Information + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {/* Advanced Settings */} + + + + + Advanced Settings + + } + key="1" + > + + Block Project + + + + + prev.isBlocked !== cur.isBlocked}> + {({ getFieldValue }) => + getFieldValue("isBlocked") ? ( + + ) : null + } + + + + + + Model-Specific Limits + + + {(fields, { add, remove }) => ( + <> + {fields.map(({ key, name, ...restField }) => ( + + + + + + + + + + + remove(name)} style={{ color: "#ef4444" }} /> + + ))} + + + + + )} + + + + + + Metadata + + + {(fields, { add, remove }) => ( + <> + {fields.map(({ key, name, ...restField }) => ( + + + + + + + + remove(name)} style={{ color: "#ef4444" }} /> + + ))} + + + + + )} + + + + + + +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx new file mode 100644 index 00000000000..40ab5045703 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx @@ -0,0 +1,211 @@ +import { useProjects, ProjectResponse } from "@/app/(dashboard)/hooks/projects/useProjects"; +import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; +import { PlusOutlined } from "@ant-design/icons"; +import { + Button, + Card, + Flex, + Input, + Layout, + Space, + Table, + Tag, + theme, + Tooltip, + Typography, +} from "antd"; +import type { ColumnsType } from "antd/es/table"; +import { LayersIcon, SearchIcon } from "lucide-react"; +import { useEffect, useMemo, useState } from "react"; +import { CreateProjectModal } from "./ProjectModals/CreateProjectModal"; + +const { Title, Text } = Typography; +const { Content } = Layout; + +export function ProjectsPage() { + const { token } = theme.useToken(); + const { data: projects, isLoading } = useProjects(); + const { data: teams } = useTeams(); + + const [isCreateModalVisible, setIsCreateModalVisible] = useState(false); + const [searchText, setSearchText] = useState(""); + const [currentPage, setCurrentPage] = useState(1); + const pageSize = 10; + + useEffect(() => { + setCurrentPage(1); + }, [searchText]); + + // Build a team_id → team_alias lookup from the teams list + const teamAliasMap = useMemo(() => { + const map = new Map(); + for (const team of teams ?? []) { + map.set(team.team_id, team.team_alias ?? team.team_id); + } + return map; + }, [teams]); + + // ---------- filtered data ---------- + const filteredProjects = useMemo(() => { + const list = projects ?? []; + if (!searchText) return list; + const lower = searchText.toLowerCase(); + return list.filter((p) => { + const alias = teamAliasMap.get(p.team_id ?? "") ?? ""; + return ( + (p.project_alias ?? "").toLowerCase().includes(lower) || + p.project_id.toLowerCase().includes(lower) || + (p.description ?? "").toLowerCase().includes(lower) || + alias.toLowerCase().includes(lower) + ); + }); + }, [projects, searchText, teamAliasMap]); + + // ---------- Ant Design columns ---------- + const columns: ColumnsType = [ + { + title: "ID", + dataIndex: "project_id", + key: "project_id", + width: 170, + render: (id: string) => ( + + + {id} + + + ), + }, + { + title: "Name", + dataIndex: "project_alias", + key: "project_alias", + sorter: (a, b) => (a.project_alias ?? "").localeCompare(b.project_alias ?? ""), + render: (alias: string | null) => alias ?? "—", + }, + { + title: "Team", + key: "team", + sorter: (a, b) => { + const aAlias = teamAliasMap.get(a.team_id ?? "") ?? ""; + const bAlias = teamAliasMap.get(b.team_id ?? "") ?? ""; + return aAlias.localeCompare(bAlias); + }, + render: (_: unknown, record: ProjectResponse) => { + const alias = teamAliasMap.get(record.team_id ?? ""); + return alias ?? record.team_id ?? "—"; + }, + }, + { + title: "Models", + key: "models", + render: (_: unknown, record: ProjectResponse) => { + const models = record.models ?? []; + return ( + 0 ? models.join(", ") : "No models"}> + + + + {models.length} + + + + ); + }, + }, + { + title: "Status", + dataIndex: "blocked", + key: "status", + render: (blocked: boolean) => ( + + {blocked ? "Blocked" : "Active"} + + ), + }, + { + title: "Created", + dataIndex: "created_at", + key: "created_at", + sorter: (a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime(), + responsive: ["lg"], + render: (date: string) => new Date(date).toLocaleDateString(), + }, + { + title: "Updated", + dataIndex: "updated_at", + key: "updated_at", + responsive: ["xl"], + render: (date: string) => new Date(date).toLocaleDateString(), + }, + ]; + + return ( + + + + + Projects + + + Manage projects within your teams + + + + + + + + } + placeholder="Search projects by name, ID, description, or team..." + style={{ maxWidth: 400 }} + value={searchText} + onChange={(e) => setSearchText(e.target.value)} + allowClear + /> + + setCurrentPage(page), + size: "small", + showTotal: (total) => `${total} projects`, + showSizeChanger: false, + }} + /> + + + setIsCreateModalVisible(false)} + /> + + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/types.ts b/ui/litellm-dashboard/src/components/Projects/types.ts new file mode 100644 index 00000000000..51429902dff --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/types.ts @@ -0,0 +1,14 @@ +export interface Project { + id: string; + name: string; + description: string; + teamId: string; + teamAlias: string; + models: string[]; + status: "active" | "blocked"; + spend: number; + createdAt: string; + createdBy: string; + updatedAt: string; + updatedBy: string; +} diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 2cbeb22ec81..bb0bd54c7a9 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -13,6 +13,7 @@ import { DatabaseOutlined, ExperimentOutlined, FileTextOutlined, + FolderOutlined, KeyOutlined, LineChartOutlined, PlayCircleOutlined, @@ -172,6 +173,23 @@ const menuGroups: MenuGroup[] = [ { groupLabel: "ACCESS CONTROL", items: [ + { + key: "teams", + page: "teams", + label: "Teams", + icon: , + }, + { + key: "projects", + page: "projects", + label: ( + + Projects + + ), + icon: , + roles: all_admin_roles, + }, { key: "users", page: "users", @@ -179,12 +197,6 @@ const menuGroups: MenuGroup[] = [ icon: , roles: all_admin_roles, }, - { - key: "teams", - page: "teams", - label: "Teams", - icon: , - }, { key: "organizations", page: "organizations", @@ -195,11 +207,7 @@ const menuGroups: MenuGroup[] = [ { key: "access-groups", page: "access-groups", - label: ( - - Access Groups - - ), + label: "Access Groups", icon: , roles: all_admin_roles, }, diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index 5e2c0ccc0b9..a910373d66d 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -21,6 +21,7 @@ export const pageDescriptions: Record = { users: "Manage internal user accounts and permissions", teams: "Create and manage teams for access control", organizations: "Manage organizations and their members", + projects: "Manage projects within teams", "access-groups": "Manage access groups for role-based permissions", budgets: "Set and monitor spending budgets", api_ref: "Browse API documentation and endpoints", From c47d21980626eded69208a75ec9a36af3c25f464 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 19:04:49 -0300 Subject: [PATCH 41/46] docs: add openrouter/anthropic/claude-opus-4.6 to main model cost map --- model_prices_and_context_window.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 66c8e55c3ae..186306b5b56 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23510,6 +23510,25 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, + "openrouter/anthropic/claude-opus-4.6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 3.75e-06, From 5628ba0831d3c0f0d4c425f2a286a1debb36ae61 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 19:05:33 -0300 Subject: [PATCH 42/46] docs: remove openrouter/anthropic/claude-opus-4.6 from backup JSON Model should only be in the main cost map, not the backup. --- ...odel_prices_and_context_window_backup.json | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 186306b5b56..66c8e55c3ae 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23510,25 +23510,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-opus-4.6": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 5e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 3.75e-06, From 8bb64574712e907186d0a76d9f26f17ffee7567f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 27 Feb 2026 15:41:03 -0800 Subject: [PATCH 43/46] [Fix] Include created_at and updated_at in /project/list response The /project/list endpoint was not returning created_at and updated_at timestamps because these fields were not defined in LiteLLM_ProjectTable. Added these fields to the model so FastAPI includes them in the response (values come from the database). This allows the UI to display project creation and last-updated times. Co-Authored-By: Claude Haiku 4.5 --- litellm/proxy/_types.py | 2 + .../test_project_endpoints_prisma.py | 75 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 28311ab1b3b..dfc2ba59d96 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2665,6 +2665,8 @@ class LiteLLM_ProjectTable(LiteLLMPydanticObjectBase): object_permission_id: Optional[str] = None created_by: str updated_by: str + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None litellm_budget_table: Optional[LiteLLM_BudgetTable] = None object_permission: Optional[LiteLLM_ObjectPermissionTable] = None diff --git a/tests/proxy_unit_tests/test_project_endpoints_prisma.py b/tests/proxy_unit_tests/test_project_endpoints_prisma.py index c98cb7efda0..77ed09a40f0 100644 --- a/tests/proxy_unit_tests/test_project_endpoints_prisma.py +++ b/tests/proxy_unit_tests/test_project_endpoints_prisma.py @@ -791,3 +791,78 @@ def test_litellm_entity_type_has_project(): assert hasattr(Litellm_EntityType, "PROJECT") assert Litellm_EntityType.PROJECT.value == "project" + + +@pytest.mark.asyncio +async def test_list_projects_returns_timestamps(): + """ + Test that /project/list returns created_at and updated_at for each project. + """ + from datetime import datetime, timezone + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy.management_endpoints.project_endpoints import list_projects + from litellm.proxy._types import LiteLLM_ProjectTable + + now = datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc) + + # Build a fake DB row that includes created_at and updated_at + fake_project = MagicMock() + fake_project.model_dump.return_value = { + "project_id": "proj-1", + "project_alias": "test-project", + "team_id": "team-1", + "created_by": "admin", + "updated_by": "admin", + "created_at": now, + "updated_at": now, + "models": [], + "spend": 0.0, + "blocked": False, + "budget_id": None, + "description": None, + "metadata": None, + "model_spend": None, + "model_rpm_limit": None, + "model_tpm_limit": None, + "object_permission_id": None, + "litellm_budget_table": None, + "object_permission": None, + } + # Make the fake row behave like a Pydantic model for FastAPI serialization + fake_project.project_id = "proj-1" + fake_project.created_at = now + fake_project.updated_at = now + + mock_prisma = MagicMock() + mock_prisma.db.litellm_projecttable.find_many = AsyncMock( + return_value=[fake_project] + ) + + with patch( + "litellm.proxy.proxy_server.prisma_client", mock_prisma + ): + response = await list_projects( + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="1234", + ), + ) + + assert len(response) == 1 + project = response[0] + assert project.created_at == now + assert project.updated_at == now + + +def test_litellm_project_table_has_timestamp_fields(): + """ + Test that LiteLLM_ProjectTable model includes created_at and updated_at fields, + so the /project/list response_model exposes them. + """ + from litellm.proxy._types import LiteLLM_ProjectTable + + fields = LiteLLM_ProjectTable.model_fields + assert "created_at" in fields, "LiteLLM_ProjectTable must have created_at field" + assert "updated_at" in fields, "LiteLLM_ProjectTable must have updated_at field" From 64c85dbc9fd3673b76693bbedf604ed6ff25d5d4 Mon Sep 17 00:00:00 2001 From: Rahul Dhanawade Date: Sat, 28 Feb 2026 05:29:37 +0530 Subject: [PATCH 44/46] Fix/claude code plugin schema (#22271) * fix: add missing LiteLLM_ClaudeCodePluginTable to schema.prisma - Claude Code Plugin Marketplace endpoints (/claude-code/marketplace.json, /claude-code/plugins) were returning 500 errors because LiteLLM_ClaudeCodePluginTable model was missing from both schema.prisma files - Prisma client was generated without this table causing AttributeError: 'Prisma' object has no attribute 'litellm_claudecodeplugintable' - Added missing model definition to root schema.prisma and litellm/proxy/schema.prisma Fixes #21310 * test: add regression test for LiteLLM_ClaudeCodePluginTable schema * fix: address greptile review - add @updatedAt, clean up test imports --- litellm/proxy/schema.prisma | 17 ++++++++++++++++- schema.prisma | 17 ++++++++++++++++- .../proxy/test_claude_code_marketplace.py | 18 ++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 tests/litellm/proxy/test_claude_code_marketplace.py diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 13461be3e7c..a5b0d930f58 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1096,4 +1096,19 @@ model LiteLLM_AccessGroupTable { created_by String? updated_at DateTime @default(now()) @updatedAt updated_by String? -} \ No newline at end of file +} +// Claude Code Plugin Marketplace table +model LiteLLM_ClaudeCodePluginTable { + id String @id @default(uuid()) + name String @unique + version String? + description String? + manifest_json String? + files_json String? @default("{}") + enabled Boolean @default(true) + created_at DateTime? @default(now()) + updated_at DateTime? @default(now()) @updatedAt + created_by String? + + @@map("LiteLLM_ClaudeCodePluginTable") +} diff --git a/schema.prisma b/schema.prisma index 34308b29ebf..bc32a8cce32 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1095,4 +1095,19 @@ model LiteLLM_AccessGroupTable { created_by String? updated_at DateTime @default(now()) @updatedAt updated_by String? -} \ No newline at end of file +} +// Claude Code Plugin Marketplace table +model LiteLLM_ClaudeCodePluginTable { + id String @id @default(uuid()) + name String @unique + version String? + description String? + manifest_json String? + files_json String? @default("{}") + enabled Boolean @default(true) + created_at DateTime? @default(now()) + updated_at DateTime? @default(now()) @updatedAt + created_by String? + + @@map("LiteLLM_ClaudeCodePluginTable") +} diff --git a/tests/litellm/proxy/test_claude_code_marketplace.py b/tests/litellm/proxy/test_claude_code_marketplace.py new file mode 100644 index 00000000000..5376e81012b --- /dev/null +++ b/tests/litellm/proxy/test_claude_code_marketplace.py @@ -0,0 +1,18 @@ +import pytest + + +@pytest.mark.asyncio +async def test_claude_code_plugin_table_schema_exists(): + + with open("schema.prisma", "r") as f: + schema = f.read() + assert "LiteLLM_ClaudeCodePluginTable" in schema, ( + "LiteLLM_ClaudeCodePluginTable model missing from schema.prisma - " + "this causes AttributeError on all /claude-code/plugins endpoints" + ) + + with open("litellm/proxy/schema.prisma", "r") as f: + proxy_schema = f.read() + assert "LiteLLM_ClaudeCodePluginTable" in proxy_schema, ( + "LiteLLM_ClaudeCodePluginTable model missing from litellm/proxy/schema.prisma" + ) From 4fa6742b01e60cd9b4d9c44e4ebc893947edfc9e Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Fri, 27 Feb 2026 16:07:29 -0800 Subject: [PATCH 45/46] Add Prometheus child_exit cleanup for gunicorn workers When a gunicorn worker exits (e.g. from max_requests recycling), its per-process prometheus .db files remain on disk. For gauges using livesum/liveall mode, this means the dead worker's last-known values persist as if the process were still alive. Wire gunicorn's child_exit hook to call mark_process_dead() so live-tracking gauges accurately reflect only running workers. --- litellm/proxy/prometheus_cleanup.py | 17 ++++++++++ litellm/proxy/proxy_cli.py | 9 ++++++ .../proxy/test_prometheus_cleanup.py | 31 ++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/prometheus_cleanup.py b/litellm/proxy/prometheus_cleanup.py index 6d935a8dd90..6353588532a 100644 --- a/litellm/proxy/prometheus_cleanup.py +++ b/litellm/proxy/prometheus_cleanup.py @@ -28,3 +28,20 @@ def wipe_directory(directory: str) -> None: verbose_proxy_logger.info( f"Prometheus cleanup: wiped {deleted} stale .db files from {directory}" ) + + +def mark_worker_exit(worker_pid: int) -> None: + """Remove prometheus .db files for a dead worker. Called by gunicorn child_exit hook.""" + if not os.environ.get("PROMETHEUS_MULTIPROC_DIR"): + return + try: + from prometheus_client import multiprocess + + multiprocess.mark_process_dead(worker_pid) + verbose_proxy_logger.info( + f"Prometheus cleanup: marked worker {worker_pid} as dead" + ) + except Exception as e: + verbose_proxy_logger.warning( + f"Failed to mark prometheus worker {worker_pid} as dead: {e}" + ) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index f5163114983..921d86c35c1 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -277,6 +277,15 @@ class ProxyInitializationHelpers: if max_requests_before_restart is not None: gunicorn_options["max_requests"] = max_requests_before_restart + # Clean up prometheus .db files when a worker exits (prevents ghost gauge values) + if os.environ.get("PROMETHEUS_MULTIPROC_DIR"): + from litellm.proxy.prometheus_cleanup import mark_worker_exit + + def child_exit(server, worker): + mark_worker_exit(worker.pid) + + gunicorn_options["child_exit"] = child_exit + if ssl_certfile_path is not None and ssl_keyfile_path is not None: print( # noqa f"\033[1;32mLiteLLM Proxy: Using SSL with certfile: {ssl_certfile_path} and keyfile: {ssl_keyfile_path}\033[0m\n" # noqa diff --git a/tests/test_litellm/proxy/test_prometheus_cleanup.py b/tests/test_litellm/proxy/test_prometheus_cleanup.py index 276f2b592db..b3d785f1133 100644 --- a/tests/test_litellm/proxy/test_prometheus_cleanup.py +++ b/tests/test_litellm/proxy/test_prometheus_cleanup.py @@ -10,7 +10,7 @@ from unittest.mock import patch import pytest -from litellm.proxy.prometheus_cleanup import wipe_directory +from litellm.proxy.prometheus_cleanup import mark_worker_exit, wipe_directory from litellm.proxy.proxy_cli import ProxyInitializationHelpers @@ -23,6 +23,35 @@ class TestWipeDirectory: assert not list(tmp_path.glob("*.db")) +class TestMarkWorkerExit: + def test_calls_mark_process_dead_when_env_set(self, tmp_path): + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark: + mark_worker_exit(12345) + mock_mark.assert_called_once_with(12345) + + def test_noop_when_env_not_set(self): + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark: + mark_worker_exit(12345) + mock_mark.assert_not_called() + + def test_exception_is_caught_and_logged(self, tmp_path): + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + with patch( + "prometheus_client.multiprocess.mark_process_dead", + side_effect=FileNotFoundError("gone"), + ) as mock_mark: + # Should not raise + mark_worker_exit(99) + mock_mark.assert_called_once_with(99) + + class TestMaybeSetupPrometheusMultiprocDir: def test_respects_existing_env_var(self, tmp_path): """When PROMETHEUS_MULTIPROC_DIR is already set, don't override it.""" From af6fe184fb14087b3b9dc7b08628695d21b7ac92 Mon Sep 17 00:00:00 2001 From: Dylan Duan Date: Fri, 27 Feb 2026 20:24:48 -0500 Subject: [PATCH 46/46] docs: update AssemblyAI docs with Universal-3 Pro, Speech Understanding, and LLM Gateway (#21130) * docs: update AssemblyAI docs with Universal-3 Pro, Speech Understanding, and LLM Gateway provider config * feat: add AssemblyAI LLM Gateway as OpenAI-compatible provider --- .../docs/pass_through/assembly_ai.md | 179 ++++++++++++++---- litellm/llms/openai_like/providers.json | 4 + .../openai_like/test_assemblyai_provider.py | 77 ++++++++ 3 files changed, 225 insertions(+), 35 deletions(-) create mode 100644 tests/litellm/llms/openai_like/test_assemblyai_provider.py diff --git a/docs/my-website/docs/pass_through/assembly_ai.md b/docs/my-website/docs/pass_through/assembly_ai.md index 4606640c5c4..c7c70639e7e 100644 --- a/docs/my-website/docs/pass_through/assembly_ai.md +++ b/docs/my-website/docs/pass_through/assembly_ai.md @@ -1,31 +1,36 @@ -# Assembly AI +# AssemblyAI -Pass-through endpoints for Assembly AI - call Assembly AI endpoints, in native format (no translation). +Pass-through endpoints for AssemblyAI - call AssemblyAI endpoints, in native format (no translation). -| Feature | Supported | Notes | +| Feature | Supported | Notes | |-------|-------|-------| | Cost Tracking | ✅ | works across all integrations | | Logging | ✅ | works across all integrations | -Supports **ALL** Assembly AI Endpoints +Supports **ALL** AssemblyAI Endpoints -[**See All Assembly AI Endpoints**](https://www.assemblyai.com/docs/api-reference) +[**See All AssemblyAI Endpoints**](https://www.assemblyai.com/docs/api-reference) - +## Supported Routes + +| AssemblyAI Service | LiteLLM Route | AssemblyAI Base URL | +|-------------------|---------------|---------------------| +| Speech-to-Text (US) | `/assemblyai/*` | `api.assemblyai.com` | +| Speech-to-Text (EU) | `/eu.assemblyai/*` | `eu.api.assemblyai.com` | ## Quick Start -Let's call the Assembly AI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts) +Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts) -1. Add Assembly AI API Key to your environment +1. Add AssemblyAI API Key to your environment ```bash export ASSEMBLYAI_API_KEY="" ``` -2. Start LiteLLM Proxy +2. Start LiteLLM Proxy ```bash litellm @@ -33,53 +38,157 @@ litellm # RUNNING on http://0.0.0.0:4000 ``` -3. Test it! +3. Test it! -Let's call the Assembly AI `/v2/transcripts` endpoint +Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts). Includes commented-out [Speech Understanding](https://www.assemblyai.com/docs/speech-understanding) features you can toggle on. ```python import assemblyai as aai -LITELLM_VIRTUAL_KEY = "sk-1234" # -LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/assemblyai" # /assemblyai +aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # /assemblyai +aai.settings.api_key = "Bearer sk-1234" # Bearer -aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}" -aai.settings.base_url = LITELLM_PROXY_BASE_URL +# Use a publicly-accessible URL +audio_file = "https://assembly.ai/wildfires.mp3" -# URL of the file to transcribe -FILE_URL = "https://assembly.ai/wildfires.mp3" +# Or use a local file: +# audio_file = "./example.mp3" -# You can also transcribe a local file by passing in a file path -# FILE_URL = './path/to/file.mp3' +config = aai.TranscriptionConfig( + speech_models=["universal-3-pro", "universal-2"], + language_detection=True, + speaker_labels=True, + # Speech understanding features + # sentiment_analysis=True, + # entity_detection=True, + # auto_chapters=True, + # summarization=True, + # summary_type=aai.SummarizationType.bullets, + # redact_pii=True, + # content_safety=True, +) -transcriber = aai.Transcriber() -transcript = transcriber.transcribe(FILE_URL) -print(transcript) -print(transcript.id) +transcript = aai.Transcriber().transcribe(audio_file, config=config) + +if transcript.status == aai.TranscriptStatus.error: + raise RuntimeError(f"Transcription failed: {transcript.error}") + +print(f"\nFull Transcript:\n\n{transcript.text}") + +# Optionally print speaker diarization results +# for utterance in transcript.utterances: +# print(f"Speaker {utterance.speaker}: {utterance.text}") ``` -## Calling Assembly AI EU endpoints +4. [Prompting with Universal-3 Pro](https://www.assemblyai.com/docs/speech-to-text/prompting) (optional) -If you want to send your request to the Assembly AI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `/eu.assemblyai` +```python +import assemblyai as aai + +aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # /assemblyai +aai.settings.api_key = "Bearer sk-1234" # Bearer + +audio_file = "https://assemblyaiassets.com/audios/verbatim.mp3" + +config = aai.TranscriptionConfig( + speech_models=["universal-3-pro", "universal-2"], + language_detection=True, + prompt="Produce a transcript suitable for conversational analysis. Every disfluency is meaningful data. Include: fillers (um, uh, er, ah, hmm, mhm, like, you know, I mean), repetitions (I I, the the), restarts (I was- I went), stutters (th-that, b-but, no-not), and informal speech (gonna, wanna, gotta)", +) + +transcript = aai.Transcriber().transcribe(audio_file, config) + +print(transcript.text) +``` + +## Calling AssemblyAI EU endpoints + +If you want to send your request to the AssemblyAI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `/eu.assemblyai` ```python import assemblyai as aai -LITELLM_VIRTUAL_KEY = "sk-1234" # -LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/eu.assemblyai" # /eu.assemblyai +aai.settings.base_url = "http://0.0.0.0:4000/eu.assemblyai" # /eu.assemblyai +aai.settings.api_key = "Bearer sk-1234" # Bearer -aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}" -aai.settings.base_url = LITELLM_PROXY_BASE_URL +# Use a publicly-accessible URL +audio_file = "https://assembly.ai/wildfires.mp3" -# URL of the file to transcribe -FILE_URL = "https://assembly.ai/wildfires.mp3" - -# You can also transcribe a local file by passing in a file path -# FILE_URL = './path/to/file.mp3' +# Or use a local file: +# audio_file = "./path/to/file.mp3" transcriber = aai.Transcriber() -transcript = transcriber.transcribe(FILE_URL) +transcript = transcriber.transcribe(audio_file) print(transcript) print(transcript.id) ``` + +## LLM Gateway + +Use AssemblyAI's [LLM Gateway](https://www.assemblyai.com/docs/llm-gateway) as an OpenAI-compatible provider — a unified API for Claude, GPT, and Gemini models with full LiteLLM logging, guardrails, and cost tracking support. + +[**See Available Models**](https://www.assemblyai.com/docs/llm-gateway#available-models) + +### Usage + +#### LiteLLM Python SDK + +```python +import litellm +import os + +os.environ["ASSEMBLYAI_API_KEY"] = "your-assemblyai-api-key" + +response = litellm.completion( + model="assemblyai/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "What is the capital of France?"}] +) + +print(response.choices[0].message.content) +``` + +#### LiteLLM Proxy + +1. Config + +```yaml +model_list: + - model_name: assemblyai/* + litellm_params: + model: assemblyai/* + api_key: os.environ/ASSEMBLYAI_API_KEY +``` + +2. Start proxy + +```bash +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +3. Test it! + +```python +import requests + +headers = { + "authorization": "Bearer sk-1234" # Bearer +} + +response = requests.post( + "http://0.0.0.0:4000/v1/chat/completions", + headers=headers, + json={ + "model": "assemblyai/claude-sonnet-4-5-20250929", + "messages": [ + {"role": "user", "content": "What is the capital of France?"} + ], + "max_tokens": 1000 + } +) + +result = response.json() +print(result["choices"][0]["message"]["content"]) +``` diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 1b1b1c2f8cc..b3125d4ad38 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -90,5 +90,9 @@ "headers": { "api-subscription-key": "{api_key}" } + }, + "assemblyai": { + "base_url": "https://llm-gateway.assemblyai.com/v1", + "api_key_env": "ASSEMBLYAI_API_KEY" } } diff --git a/tests/litellm/llms/openai_like/test_assemblyai_provider.py b/tests/litellm/llms/openai_like/test_assemblyai_provider.py new file mode 100644 index 00000000000..7eee810b271 --- /dev/null +++ b/tests/litellm/llms/openai_like/test_assemblyai_provider.py @@ -0,0 +1,77 @@ +""" +Unit tests for the AssemblyAI LLM Gateway OpenAI-like provider. +""" + +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) +) + +from litellm.llms.openai_like.dynamic_config import create_config_class +from litellm.llms.openai_like.json_loader import JSONProviderRegistry + +ASSEMBLYAI_BASE_URL = "https://llm-gateway.assemblyai.com/v1" + + +def _get_config(): + provider = JSONProviderRegistry.get("assemblyai") + assert provider is not None + config_class = create_config_class(provider) + return config_class() + + +def test_assemblyai_provider_registered(): + provider = JSONProviderRegistry.get("assemblyai") + assert provider is not None + assert provider.base_url == ASSEMBLYAI_BASE_URL + assert provider.api_key_env == "ASSEMBLYAI_API_KEY" + + +def test_assemblyai_resolves_env_api_key(monkeypatch): + config = _get_config() + monkeypatch.setenv("ASSEMBLYAI_API_KEY", "test-key") + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == ASSEMBLYAI_BASE_URL + assert api_key == "test-key" + + +def test_assemblyai_complete_url_appends_endpoint(): + config = _get_config() + url = config.get_complete_url( + api_base=ASSEMBLYAI_BASE_URL, + api_key="test-key", + model="assemblyai/claude-sonnet-4-5-20250929", + optional_params={}, + litellm_params={}, + stream=False, + ) + assert url == f"{ASSEMBLYAI_BASE_URL}/chat/completions" + + +def test_assemblyai_provider_resolution(): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="assemblyai/claude-sonnet-4-5-20250929", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "claude-sonnet-4-5-20250929" + assert provider == "assemblyai" + assert api_base == ASSEMBLYAI_BASE_URL + + +def test_assemblyai_provider_config_manager(): + from litellm import LlmProviders + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_chat_config( + model="claude-sonnet-4-5-20250929", provider=LlmProviders.ASSEMBLYAI + ) + + assert config is not None + assert config.custom_llm_provider == "assemblyai"