From 0bc609affdc09c4071c72f4035591289685af8e4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 11:23:47 +0530 Subject: [PATCH 01/83] fix(vertex-ai): support batch cancel via Vertex API Add Vertex batch cancellation support in LiteLLM batch APIs, route proxy cancel fallback using request provider headers, and return post-cancel batch state via retrieve to keep response shape compatible. Made-with: Cursor --- litellm/batches/main.py | 32 ++++- litellm/llms/vertex_ai/batches/handler.py | 113 ++++++++++++++++++ litellm/proxy/batches_endpoints/endpoints.py | 14 ++- .../test_vertex_ai_batch_transformation.py | 78 ++++++++++++ 4 files changed, 229 insertions(+), 8 deletions(-) diff --git a/litellm/batches/main.py b/litellm/batches/main.py index e176dc42921..36093d071ba 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -884,7 +884,7 @@ def list_batches( async def acancel_batch( batch_id: str, model: Optional[str] = None, - custom_llm_provider: Literal["openai", "azure"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", metadata: Optional[Dict[str, str]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -930,7 +930,7 @@ async def acancel_batch( def cancel_batch( batch_id: str, model: Optional[str] = None, - custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai", + custom_llm_provider: Union[Literal["openai", "azure", "vertex_ai"], str] = "openai", metadata: Optional[Dict[str, str]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -1047,9 +1047,35 @@ def cancel_batch( cancel_batch_data=_cancel_batch_request, litellm_params=litellm_params, ) + elif custom_llm_provider == "vertex_ai": + api_base = optional_params.api_base or "" + vertex_ai_project = ( + optional_params.vertex_project + or litellm.vertex_project + or get_secret_str("VERTEXAI_PROJECT") + ) + vertex_ai_location = ( + optional_params.vertex_location + or litellm.vertex_location + or get_secret_str("VERTEXAI_LOCATION") + ) + vertex_credentials = optional_params.vertex_credentials or get_secret_str( + "VERTEXAI_CREDENTIALS" + ) + + response = vertex_ai_batches_instance.cancel_batch( + _is_async=_is_async, + batch_id=batch_id, + api_base=api_base, + vertex_project=vertex_ai_project, + vertex_location=vertex_ai_location, + vertex_credentials=vertex_credentials, + timeout=timeout, + max_retries=optional_params.max_retries, + ) else: raise litellm.exceptions.BadRequestError( - message="LiteLLM doesn't support {} for 'cancel_batch'. Only 'openai' and 'azure' are supported.".format( + message="LiteLLM doesn't support {} for 'cancel_batch'. Only 'openai', 'azure', and 'vertex_ai' are supported.".format( custom_llm_provider ), model="n/a", diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index f0b181c9a61..a24bfd89f07 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -376,3 +376,116 @@ class VertexAIBatchPrediction(VertexLLM): response=_json_response ) return vertex_batch_response + + def cancel_batch( + self, + _is_async: bool, + batch_id: str, + api_base: Optional[str], + vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES], + vertex_project: Optional[str], + vertex_location: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + ) -> Union[LiteLLMBatch, Coroutine[Any, Any, LiteLLMBatch]]: + sync_handler = _get_httpx_client() + + access_token, project_id = self._ensure_access_token( + credentials=vertex_credentials, + project_id=vertex_project, + custom_llm_provider="vertex_ai", + ) + + default_api_base = self.create_vertex_batch_url( + vertex_location=vertex_location or "us-central1", + vertex_project=vertex_project or project_id, + ) + + default_api_base = f"{default_api_base}/{batch_id}:cancel" + + if len(default_api_base.split(":")) > 1: + endpoint = default_api_base.split(":")[-1] + else: + endpoint = "" + + _, api_base = self._check_custom_proxy( + api_base=api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint=endpoint, + stream=None, + auth_header=None, + url=default_api_base, + model=None, + vertex_project=vertex_project or project_id, + vertex_location=vertex_location or "us-central1", + vertex_api_version="v1", + ) + + headers = { + "Content-Type": "application/json; charset=utf-8", + "Authorization": f"Bearer {access_token}", + } + + if _is_async is True: + return self._async_cancel_batch( + api_base=api_base, + retrieve_api_base=api_base.rsplit(":cancel", 1)[0], + headers=headers, + ) + + response = sync_handler.post( + url=api_base, + headers=headers, + data=json.dumps({}), + ) + + if response.status_code != 200: + raise Exception(f"Error: {response.status_code} {response.text}") + + retrieve_response = sync_handler.get( + url=api_base.rsplit(":cancel", 1)[0], + headers=headers, + ) + if retrieve_response.status_code != 200: + raise Exception( + f"Error: {retrieve_response.status_code} {retrieve_response.text}" + ) + + _json_response = retrieve_response.json() + vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response( + response=_json_response + ) + return vertex_batch_response + + async def _async_cancel_batch( + self, + api_base: str, + retrieve_api_base: str, + headers: Dict[str, str], + ) -> LiteLLMBatch: + client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.VERTEX_AI, + ) + response = await client.post( + url=api_base, + headers=headers, + data=json.dumps({}), + ) + if response.status_code != 200: + raise Exception(f"Error: {response.status_code} {response.text}") + + retrieve_response = await client.get( + url=retrieve_api_base, + headers=headers, + ) + if retrieve_response.status_code != 200: + raise Exception( + f"Error: {retrieve_response.status_code} {retrieve_response.text}" + ) + + _json_response = retrieve_response.json() + vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response( + response=_json_response + ) + return vertex_batch_response diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 740e63b7f17..06254b57fc5 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -5,7 +5,7 @@ ###################################################################### import asyncio -from typing import Dict, Optional, cast +from typing import Any, Dict, Optional, cast from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response @@ -654,7 +654,7 @@ async def list_batches( managed_files_obj, "list_user_batches" ): verbose_proxy_logger.debug("Using managed objects table for batch listing") - response = await managed_files_obj.list_user_batches( + response = await cast(Any, managed_files_obj).list_user_batches( user_api_key_dict=user_api_key_dict, limit=limit, after=after, @@ -685,8 +685,9 @@ async def list_batches( # Encode batch IDs in the list response so clients can use # them for retrieve/cancel/file downloads through the proxy. - if response and hasattr(response, "data") and response.data: - for batch in response.data: + response_data = getattr(response, "data", None) + if response_data: + for batch in response_data: encode_batch_response_ids(batch, model=model_param) verbose_proxy_logger.debug(f"Listed batches using model: {model_param}") @@ -896,7 +897,10 @@ async def cancel_batch( # SCENARIO 3: Fallback to custom_llm_provider (uses env variables) else: custom_llm_provider = ( - provider or data.pop("custom_llm_provider", None) or "openai" + provider + or get_custom_llm_provider_from_request_headers(request=request) + or data.pop("custom_llm_provider", None) + or "openai" ) # Extract batch_id from data to avoid "multiple values for keyword argument" error # data was cast from CancelBatchRequest which already contains batch_id diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py index 1aab74ddc26..d27cd8ba8a7 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py @@ -1,3 +1,9 @@ +from unittest.mock import MagicMock, patch + +import pytest + +import litellm +from litellm.llms.vertex_ai.batches.handler import VertexAIBatchPrediction from litellm.llms.vertex_ai.batches.transformation import VertexAIBatchTransformation @@ -36,3 +42,75 @@ def test_output_file_id_falls_back_to_output_uri_prefix_with_predictions_jsonl() output_file_id == "gs://test-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-pro/prediction-model-456/predictions.jsonl" ) + + +@pytest.mark.asyncio +async def test_vertex_ai_cancel_batch(): + """Test that vertex_ai cancel_batch calls the correct API endpoint""" + handler = VertexAIBatchPrediction(gcs_bucket_name="test-bucket") + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "name": "projects/test-project/locations/us-central1/batchPredictionJobs/123456", + "state": "JOB_STATE_CANCELLING", + "createTime": "2024-03-17T10:00:00.000000Z", + "inputConfig": { + "gcsSource": { + "uris": ["gs://test-bucket/input.jsonl"] + } + }, + "outputConfig": { + "gcsDestination": { + "outputUriPrefix": "gs://test-bucket/output" + } + } + } + + with patch("litellm.llms.vertex_ai.batches.handler._get_httpx_client") as mock_client: + mock_client.return_value.post.return_value = mock_response + mock_client.return_value.get.return_value = mock_response + + with patch.object(handler, "_ensure_access_token") as mock_auth: + mock_auth.return_value = ("fake-token", "test-project") + + response = handler.cancel_batch( + _is_async=False, + batch_id="123456", + api_base=None, + vertex_credentials=None, + vertex_project="test-project", + vertex_location="us-central1", + timeout=600.0, + max_retries=None, + ) + + assert response.id == "123456" + assert response.status == "cancelling" + + mock_client.return_value.post.assert_called_once() + mock_client.return_value.get.assert_called_once() + call_args = mock_client.return_value.post.call_args + assert ":cancel" in call_args.kwargs["url"] + + +@pytest.mark.asyncio +async def test_litellm_cancel_batch_vertex_ai(): + """Test that litellm.cancel_batch works with vertex_ai provider""" + mock_response = MagicMock() + mock_response.id = "batch_123" + mock_response.status = "cancelling" + + with patch.object(litellm.batches.main, "vertex_ai_batches_instance") as mock_instance: + mock_instance.cancel_batch.return_value = mock_response + + response = litellm.cancel_batch( + batch_id="batch_123", + custom_llm_provider="vertex_ai", + vertex_project="test-project", + vertex_location="us-central1", + ) + + assert mock_instance.cancel_batch.called + assert response.id == "batch_123" + assert response.status == "cancelling" From d8e3abf3cef95365fa6b1cc0af68e32a33319bd1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 11:34:50 +0530 Subject: [PATCH 02/83] fix(vertex-ai): apply review updates for batch cancel Incorporate follow-up changes to Vertex batch cancel handling and proxy provider resolution, including config updates used for local verification. Made-with: Cursor --- litellm/llms/vertex_ai/batches/handler.py | 3 +- litellm/proxy/batches_endpoints/endpoints.py | 1 + proxy_server_config.yaml | 231 +------------------ 3 files changed, 4 insertions(+), 231 deletions(-) diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index a24bfd89f07..c7b9287c080 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -388,8 +388,6 @@ class VertexAIBatchPrediction(VertexLLM): timeout: Union[float, httpx.Timeout], max_retries: Optional[int], ) -> Union[LiteLLMBatch, Coroutine[Any, Any, LiteLLMBatch]]: - sync_handler = _get_httpx_client() - access_token, project_id = self._ensure_access_token( credentials=vertex_credentials, project_id=vertex_project, @@ -434,6 +432,7 @@ class VertexAIBatchPrediction(VertexLLM): headers=headers, ) + sync_handler = _get_httpx_client() response = sync_handler.post( url=api_base, headers=headers, diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 06254b57fc5..9ce1b6e9168 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -899,6 +899,7 @@ async def cancel_batch( custom_llm_provider = ( provider or get_custom_llm_provider_from_request_headers(request=request) + or get_custom_llm_provider_from_request_query(request=request) or data.pop("custom_llm_provider", None) or "openai" ) diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index 5d3d810926a..6e48af021a4 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -1,231 +1,4 @@ model_list: - - model_name: gpt-3.5-turbo-end-user-test + - model_name: gemini-2.5-pro litellm_params: - model: gpt-3.5-turbo - region_name: "eu" - model_info: - id: "1" - - model_name: gpt-3.5-turbo-end-user-test - litellm_params: - model: openai/gpt-4.1-mini - api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - - model_name: gpt-3.5-turbo - litellm_params: - model: openai/gpt-4.1-mini - api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - - model_name: gpt-3.5-turbo-large - litellm_params: - model: "gpt-3.5-turbo-1106" - api_key: os.environ/OPENAI_API_KEY - rpm: 480 - timeout: 300 - stream_timeout: 60 - - model_name: gpt-4 - litellm_params: - model: openai/gpt-4.1-mini - api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - rpm: 480 - timeout: 300 - stream_timeout: 60 - - model_name: sagemaker-completion-model - litellm_params: - model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4 - input_cost_per_second: 0.000420 - - model_name: text-embedding-ada-002 - litellm_params: - model: openai/text-embedding-ada-002 - api_key: os.environ/OPENAI_API_KEY - model_info: - mode: embedding - base_model: text-embedding-ada-002 - - model_name: dall-e-2 # some tests use dall-e-2 which is now deprecated, alias to dall-e-3 - litellm_params: - model: openai/dall-e-3 - - model_name: openai-dall-e-3 - litellm_params: - model: dall-e-3 - - model_name: fake-openai-endpoint - litellm_params: - model: openai/gpt-3.5-turbo - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - - model_name: fake-openai-endpoint-2 - litellm_params: - model: openai/my-fake-model - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - stream_timeout: 0.001 - rpm: 1 - - model_name: fake-openai-endpoint-3 - litellm_params: - model: openai/my-fake-model - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - stream_timeout: 0.001 - rpm: 1000 - - model_name: fake-openai-endpoint-4 - litellm_params: - model: openai/my-fake-model - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - num_retries: 50 - - model_name: fake-openai-endpoint-3 - litellm_params: - model: openai/my-fake-model-2 - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - stream_timeout: 0.001 - rpm: 1000 - - model_name: bad-model - litellm_params: - model: openai/bad-model - api_key: os.environ/OPENAI_API_KEY - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - mock_timeout: True - timeout: 60 - rpm: 1000 - model_info: - health_check_timeout: 1 - - model_name: good-model - litellm_params: - model: openai/bad-model - api_key: os.environ/OPENAI_API_KEY - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - rpm: 1000 - model_info: - health_check_timeout: 1 - - model_name: "*" - litellm_params: - model: openai/* - api_key: os.environ/OPENAI_API_KEY - - model_name: realtime-v1 - litellm_params: - model: azure/gpt-realtime-20250828-standard - api_version: "2025-08-28" - realtime_protocol: GA # Possible values: "GA"/ "v1", "beta" - - - model_name: realtime-beta - litellm_params: - model: azure/gpt-realtime-20250828-standard - api_version: 2025-04-01-preview - - - # provider specific wildcard routing - - model_name: "anthropic/*" - litellm_params: - model: "anthropic/*" - api_key: os.environ/ANTHROPIC_API_KEY - - model_name: "bedrock/*" - litellm_params: - model: "bedrock/*" - - model_name: "groq/*" - litellm_params: - model: "groq/*" - api_key: os.environ/GROQ_API_KEY - - model_name: mistral-embed - litellm_params: - model: mistral/mistral-embed - - model_name: gpt-instruct # [PROD TEST] - tests if `/health` automatically infers this to be a text completion model - litellm_params: - model: text-completion-openai/gpt-3.5-turbo-instruct - - model_name: fake-openai-endpoint-5 - litellm_params: - model: openai/my-fake-model - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - timeout: 1 - - model_name: badly-configured-openai-endpoint - litellm_params: - model: openai/my-fake-model - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.appxxxx/ - - model_name: gemini-1.5-flash - litellm_params: - model: gemini/gemini-1.5-flash - api_key: os.environ/GOOGLE_API_KEY - - model_name: gpt-4o - litellm_params: - model: gpt-4o - api_key: os.environ/OPENAI_API_KEY - - -litellm_settings: - # set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production - drop_params: True - success_callback: ["prometheus"] - # max_budget: 100 - # budget_duration: 30d - num_retries: 5 - request_timeout: 600 - telemetry: False - context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}] - default_team_settings: - - team_id: team-1 - success_callback: ["langfuse"] - failure_callback: ["langfuse"] - langfuse_public_key: os.environ/LANGFUSE_PROJECT1_PUBLIC # Project 1 - langfuse_secret: os.environ/LANGFUSE_PROJECT1_SECRET # Project 1 - - team_id: team-2 - success_callback: ["langfuse"] - failure_callback: ["langfuse"] - langfuse_public_key: os.environ/LANGFUSE_PROJECT2_PUBLIC # Project 2 - langfuse_secret: os.environ/LANGFUSE_PROJECT2_SECRET # Project 2 - langfuse_host: https://us.cloud.langfuse.com - # cache: true # [OPTIONAL] use for caching responses - # enable_caching_on_provider_specific_optional_params: True # Include provider-specific params in cache keys - # cache_params: # And for shared health check - # type: redis - # host: localhost - # port: 6379 - -# For /fine_tuning/jobs endpoints -finetune_settings: - - custom_llm_provider: azure - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - api_version: "2023-03-15-preview" - - custom_llm_provider: openai - api_key: os.environ/OPENAI_API_KEY - -# for /files endpoints -files_settings: - - custom_llm_provider: azure - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - api_version: "2023-03-15-preview" - - custom_llm_provider: openai - api_key: os.environ/OPENAI_API_KEY - -router_settings: - routing_strategy: usage-based-routing-v2 - redis_host: os.environ/REDIS_HOST - redis_password: os.environ/REDIS_PASSWORD - redis_port: os.environ/REDIS_PORT - enable_pre_call_checks: true - model_group_alias: {"my-special-fake-model-alias-name": "fake-openai-endpoint-3"} - -general_settings: - master_key: sk-1234 # [OPTIONAL] Use to enforce auth on proxy. See - https://docs.litellm.ai/docs/proxy/virtual_keys - store_model_in_db: True - proxy_budget_rescheduler_min_time: 60 - proxy_budget_rescheduler_max_time: 64 - proxy_batch_write_at: 1 - database_connection_pool_limit: 10 - # background_health_checks: true - # use_shared_health_check: true - # health_check_interval: 30 - # database_url: "postgresql://:@:/" # [OPTIONAL] use for token-based auth to proxy - - pass_through_endpoints: - - path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server - target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to - headers: # headers to forward to this URL - content-type: application/json # (Optional) Extra Headers to pass to this endpoint - accept: application/json - forward_headers: True - -# environment_variables: - # settings for using redis caching - # REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com - # REDIS_PORT: "16337" - # REDIS_PASSWORD: \ No newline at end of file + model: vertex_ai/gemini-2.5-pro \ No newline at end of file From 37b7a7fb576279a41817ef8239d2e24b79e56f7b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 11:36:39 +0530 Subject: [PATCH 03/83] chore(config): restore proxy_server_config.yaml Revert local test-only proxy config edits so the PR does not include unrelated configuration changes. Made-with: Cursor --- proxy_server_config.yaml | 231 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 2 deletions(-) diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index 6e48af021a4..5d3d810926a 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -1,4 +1,231 @@ model_list: - - model_name: gemini-2.5-pro + - model_name: gpt-3.5-turbo-end-user-test litellm_params: - model: vertex_ai/gemini-2.5-pro \ No newline at end of file + model: gpt-3.5-turbo + region_name: "eu" + model_info: + id: "1" + - model_name: gpt-3.5-turbo-end-user-test + litellm_params: + model: openai/gpt-4.1-mini + api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-4.1-mini + api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault + - model_name: gpt-3.5-turbo-large + litellm_params: + model: "gpt-3.5-turbo-1106" + api_key: os.environ/OPENAI_API_KEY + rpm: 480 + timeout: 300 + stream_timeout: 60 + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4.1-mini + api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault + rpm: 480 + timeout: 300 + stream_timeout: 60 + - model_name: sagemaker-completion-model + litellm_params: + model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4 + input_cost_per_second: 0.000420 + - model_name: text-embedding-ada-002 + litellm_params: + model: openai/text-embedding-ada-002 + api_key: os.environ/OPENAI_API_KEY + model_info: + mode: embedding + base_model: text-embedding-ada-002 + - model_name: dall-e-2 # some tests use dall-e-2 which is now deprecated, alias to dall-e-3 + litellm_params: + model: openai/dall-e-3 + - model_name: openai-dall-e-3 + litellm_params: + model: dall-e-3 + - model_name: fake-openai-endpoint + litellm_params: + model: openai/gpt-3.5-turbo + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + - model_name: fake-openai-endpoint-2 + litellm_params: + model: openai/my-fake-model + api_key: my-fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + stream_timeout: 0.001 + rpm: 1 + - model_name: fake-openai-endpoint-3 + litellm_params: + model: openai/my-fake-model + api_key: my-fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + stream_timeout: 0.001 + rpm: 1000 + - model_name: fake-openai-endpoint-4 + litellm_params: + model: openai/my-fake-model + api_key: my-fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + num_retries: 50 + - model_name: fake-openai-endpoint-3 + litellm_params: + model: openai/my-fake-model-2 + api_key: my-fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + stream_timeout: 0.001 + rpm: 1000 + - model_name: bad-model + litellm_params: + model: openai/bad-model + api_key: os.environ/OPENAI_API_KEY + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + mock_timeout: True + timeout: 60 + rpm: 1000 + model_info: + health_check_timeout: 1 + - model_name: good-model + litellm_params: + model: openai/bad-model + api_key: os.environ/OPENAI_API_KEY + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + rpm: 1000 + model_info: + health_check_timeout: 1 + - model_name: "*" + litellm_params: + model: openai/* + api_key: os.environ/OPENAI_API_KEY + - model_name: realtime-v1 + litellm_params: + model: azure/gpt-realtime-20250828-standard + api_version: "2025-08-28" + realtime_protocol: GA # Possible values: "GA"/ "v1", "beta" + + - model_name: realtime-beta + litellm_params: + model: azure/gpt-realtime-20250828-standard + api_version: 2025-04-01-preview + + + # provider specific wildcard routing + - model_name: "anthropic/*" + litellm_params: + model: "anthropic/*" + api_key: os.environ/ANTHROPIC_API_KEY + - model_name: "bedrock/*" + litellm_params: + model: "bedrock/*" + - model_name: "groq/*" + litellm_params: + model: "groq/*" + api_key: os.environ/GROQ_API_KEY + - model_name: mistral-embed + litellm_params: + model: mistral/mistral-embed + - model_name: gpt-instruct # [PROD TEST] - tests if `/health` automatically infers this to be a text completion model + litellm_params: + model: text-completion-openai/gpt-3.5-turbo-instruct + - model_name: fake-openai-endpoint-5 + litellm_params: + model: openai/my-fake-model + api_key: my-fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + timeout: 1 + - model_name: badly-configured-openai-endpoint + litellm_params: + model: openai/my-fake-model + api_key: my-fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.appxxxx/ + - model_name: gemini-1.5-flash + litellm_params: + model: gemini/gemini-1.5-flash + api_key: os.environ/GOOGLE_API_KEY + - model_name: gpt-4o + litellm_params: + model: gpt-4o + api_key: os.environ/OPENAI_API_KEY + + +litellm_settings: + # set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production + drop_params: True + success_callback: ["prometheus"] + # max_budget: 100 + # budget_duration: 30d + num_retries: 5 + request_timeout: 600 + telemetry: False + context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}] + default_team_settings: + - team_id: team-1 + success_callback: ["langfuse"] + failure_callback: ["langfuse"] + langfuse_public_key: os.environ/LANGFUSE_PROJECT1_PUBLIC # Project 1 + langfuse_secret: os.environ/LANGFUSE_PROJECT1_SECRET # Project 1 + - team_id: team-2 + success_callback: ["langfuse"] + failure_callback: ["langfuse"] + langfuse_public_key: os.environ/LANGFUSE_PROJECT2_PUBLIC # Project 2 + langfuse_secret: os.environ/LANGFUSE_PROJECT2_SECRET # Project 2 + langfuse_host: https://us.cloud.langfuse.com + # cache: true # [OPTIONAL] use for caching responses + # enable_caching_on_provider_specific_optional_params: True # Include provider-specific params in cache keys + # cache_params: # And for shared health check + # type: redis + # host: localhost + # port: 6379 + +# For /fine_tuning/jobs endpoints +finetune_settings: + - custom_llm_provider: azure + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: "2023-03-15-preview" + - custom_llm_provider: openai + api_key: os.environ/OPENAI_API_KEY + +# for /files endpoints +files_settings: + - custom_llm_provider: azure + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: "2023-03-15-preview" + - custom_llm_provider: openai + api_key: os.environ/OPENAI_API_KEY + +router_settings: + routing_strategy: usage-based-routing-v2 + redis_host: os.environ/REDIS_HOST + redis_password: os.environ/REDIS_PASSWORD + redis_port: os.environ/REDIS_PORT + enable_pre_call_checks: true + model_group_alias: {"my-special-fake-model-alias-name": "fake-openai-endpoint-3"} + +general_settings: + master_key: sk-1234 # [OPTIONAL] Use to enforce auth on proxy. See - https://docs.litellm.ai/docs/proxy/virtual_keys + store_model_in_db: True + proxy_budget_rescheduler_min_time: 60 + proxy_budget_rescheduler_max_time: 64 + proxy_batch_write_at: 1 + database_connection_pool_limit: 10 + # background_health_checks: true + # use_shared_health_check: true + # health_check_interval: 30 + # database_url: "postgresql://:@:/" # [OPTIONAL] use for token-based auth to proxy + + pass_through_endpoints: + - path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server + target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to + headers: # headers to forward to this URL + content-type: application/json # (Optional) Extra Headers to pass to this endpoint + accept: application/json + forward_headers: True + +# environment_variables: + # settings for using redis caching + # REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com + # REDIS_PORT: "16337" + # REDIS_PASSWORD: \ No newline at end of file From c7352515707fbcc83907000b5487d4f5162bf1d2 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 11:41:44 +0530 Subject: [PATCH 04/83] =?UTF-8?q?feat(responses):=20file=5Fsearch=20suppor?= =?UTF-8?q?t=20=E2=80=94=20Phase=201=20native=20passthrough=20+=20Phase=20?= =?UTF-8?q?2=20emulated=20fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 (native passthrough): - _decode_vector_store_ids_in_tools(): decode LiteLLM-managed unified vector_store_ids to provider-native IDs in file_search tools - Split update_responses_tools_with_model_file_ids() into decode pass (always runs) + code_interpreter mapping pass (guarded) - BaseResponsesAPIConfig.supports_native_file_search() → False by default; OpenAIResponsesAPIConfig overrides to True - ManagedFiles.async_pre_call_hook(): batch team-level access check for unified vector_store_ids in file_search tools (no N+1) - Docs: file_search section in response_api.md Phase 2 (emulated fallback for non-native providers): - litellm/responses/file_search/emulated_handler.py: converts file_search tool → function tool, intercepts tool call, runs asearch(), makes follow-up call, synthesizes OpenAI-format output (file_search_call + message + file_citation annotations) - responses/main.py: routes to emulated handler when provider doesn't support file_search natively Tests: 41 unit tests across 8 families (A-H) in test_file_search_responses.py Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/my-website/docs/response_api.md | 129 ++++ .../proxy/hooks/managed_files.py | 106 ++- .../prompt_templates/common_utils.py | 60 +- .../llms/base_llm/responses/transformation.py | 8 + .../llms/openai/responses/transformation.py | 3 + litellm/responses/file_search/__init__.py | 0 .../responses/file_search/emulated_handler.py | 431 +++++++++++ litellm/responses/main.py | 53 ++ .../llms/test_file_search_responses.py | 684 ++++++++++++++++++ 9 files changed, 1467 insertions(+), 7 deletions(-) create mode 100644 litellm/responses/file_search/__init__.py create mode 100644 litellm/responses/file_search/emulated_handler.py create mode 100644 tests/test_litellm/llms/test_file_search_responses.py diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index fb55ae9f9d0..183b3399009 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -1556,6 +1556,135 @@ curl -X POST "http://localhost:4000/v1/responses" \ }' ``` +## File Search (Vector Stores) + +The **file_search** tool lets the model search your vector stores and cite retrieved content in its answer (OpenAI Responses API format). Pass `tools=[{"type": "file_search", "vector_store_ids": [...]}]`. The response includes a `file_search_call` output item and `file_citation` annotations on the answer text. + +**Supported providers:** `openai`, `azure` (native). Other providers will receive an `UnsupportedParamsError` until the emulated-fallback path is available. + +:::note +If you are using LiteLLM-managed vector stores (created via `/v1/vector_stores`), pass the LiteLLM vector store ID directly — LiteLLM automatically decodes it to the provider-native ID before sending the request. +::: + +### Python SDK + +```python showLineNumbers title="File search with LiteLLM Python SDK" +import litellm + +response = litellm.responses( + model="openai/gpt-4.1", + input="What is deep research?", + tools=[{ + "type": "file_search", + "vector_store_ids": ["vs_abc123"] # native or LiteLLM-managed vector store ID + }], +) + +# Output contains a file_search_call item followed by the answer with citations +for item in response.output: + if item.type == "file_search_call": + print("Queries:", item.queries) + elif item.type == "message": + for block in item.content: + print(block.text) + for ann in block.annotations: + print(f" ↳ {ann.filename} (file_id={ann.file_id})") +``` + +#### Response Format + +```json +{ + "output": [ + { + "type": "file_search_call", + "id": "fs_67c09ccea8c48191ade9367e3ba71515", + "status": "completed", + "queries": ["What is deep research?"], + "search_results": null + }, + { + "id": "msg_67c09cd3091c819185af2be5d13d87de", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Deep research is a capability that allows for extensive inquiry ...", + "annotations": [ + { + "type": "file_citation", + "index": 992, + "file_id": "file-2dtbBZdjtDKS8eqWxqbgDi", + "filename": "deep_research_blog.pdf" + } + ] + } + ] + } + ] +} +``` + +### LiteLLM Proxy (AI Gateway) + +**OpenAI Python SDK (proxy as base_url):** + +```python showLineNumbers title="File search via LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-proxy-api-key", +) + +response = client.responses.create( + model="openai/gpt-4.1", + input="Summarise the Q3 earnings report.", + tools=[{ + "type": "file_search", + "vector_store_ids": ["vs_abc123"] + }], +) +``` + +**curl:** + +```bash title="File search via curl to LiteLLM Proxy" +curl -X POST "http://localhost:4000/v1/responses" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "openai/gpt-4.1", + "input": "Summarise the Q3 earnings report.", + "tools": [{"type": "file_search", "vector_store_ids": ["vs_abc123"]}] + }' +``` + +### Using LiteLLM-Managed Vector Stores + +If you created a vector store through LiteLLM (`POST /v1/vector_stores/new`), use the returned `vector_store_id` directly. LiteLLM decodes the unified ID to the provider-native vector store ID automatically. + +```python showLineNumbers title="File search with LiteLLM-managed vector store" +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:4000", api_key="your-proxy-api-key") + +# vector_store_id returned by POST /v1/vector_stores/new +managed_vs_id = "bGl0ZWxsbV9wcm94eTo..." # LiteLLM-managed ID + +response = client.responses.create( + model="openai/gpt-4.1", + input="What does the documentation say about authentication?", + tools=[{"type": "file_search", "vector_store_ids": [managed_vs_id]}], +) +``` + +LiteLLM will: +1. Verify the calling team has access to the vector store. +2. Decode the managed ID to the provider-native vector store ID. +3. Forward the request to the provider unchanged. + ## Session Management LiteLLM Proxy supports session management for all supported models. This allows you to store and fetch conversation history (state) in LiteLLM Proxy. diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 5530054170c..351fe05755f 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -29,7 +29,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( get_models_from_unified_file_id, normalize_mime_type_for_provider, ) -from litellm.types.llms.openai import ( +from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue] AllMessageValues, AsyncCursorPage, ChatCompletionFileObject, @@ -442,25 +442,33 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value: # Handle managed files in responses API input and tools file_ids = [] - + # Extract file IDs from input parameter input_data = data.get("input") if input_data: file_ids.extend(self.get_file_ids_from_responses_input(input_data)) - + # Extract file IDs from tools parameter (e.g., code_interpreter container) tools = data.get("tools") if tools: file_ids.extend(self.get_file_ids_from_responses_tools(tools)) - + if file_ids: # Check user has access to all managed files await self.check_file_ids_access(file_ids, user_api_key_dict) - + model_file_id_mapping = await self.get_model_file_id_mapping( file_ids, user_api_key_dict.parent_otel_span ) data["model_file_id_mapping"] = model_file_id_mapping + + # Check access for file_search vector_store_ids + if tools: + unified_vs_ids = self.get_vector_store_ids_from_file_search_tools(tools) + if unified_vs_ids: + await self.check_vector_store_ids_access( + unified_vs_ids, user_api_key_dict + ) elif call_type == CallTypes.afile_content.value: retrieve_file_id = cast(Optional[str], data.get("file_id")) potential_file_id = ( @@ -704,6 +712,92 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return file_ids + def get_vector_store_ids_from_file_search_tools( + self, tools: List[Dict[str, Any]] + ) -> List[str]: + """ + Extract unified vector_store_ids from file_search tools. + + Only returns IDs that are LiteLLM-managed (base64 unified IDs). + Native provider IDs are skipped — they have no LiteLLM access record. + """ + from litellm.llms.base_llm.managed_resources.utils import ( + is_base64_encoded_unified_id, + ) + + vs_ids: List[str] = [] + if not isinstance(tools, list): + return vs_ids + + for tool in tools: + if not isinstance(tool, dict) or tool.get("type") != "file_search": + continue + vector_store_ids = tool.get("vector_store_ids") + if not isinstance(vector_store_ids, list): + continue + for vs_id in vector_store_ids: + if isinstance(vs_id, str) and is_base64_encoded_unified_id(vs_id): + vs_ids.append(vs_id) + + return vs_ids + + async def check_vector_store_ids_access( + self, + vector_store_ids: List[str], + user_api_key_dict: UserAPIKeyAuth, + ) -> None: + """ + Verify the caller's team can access each LiteLLM-managed vector store. + + Batch-fetches vector stores from DB and checks team_id. + Raises HTTPException(403) on the first access violation. + Non-managed (native) IDs should already be filtered out before calling this. + """ + from litellm.llms.base_llm.managed_resources.utils import ( + extract_unified_uuid_from_unified_id, + ) + from litellm.proxy.proxy_server import prisma_client + + if not vector_store_ids or prisma_client is None: + return + + # Map each unified ID to its internal UUID for a single batch DB fetch + uuid_to_unified: Dict[str, str] = {} + for vs_id in vector_store_ids: + uuid = extract_unified_uuid_from_unified_id(vs_id) + if uuid: + uuid_to_unified[uuid] = vs_id + + if not uuid_to_unified: + return + + rows = await prisma_client.db.litellm_managedvectorstorestable.find_many( + where={"vector_store_id": {"in": list(uuid_to_unified.keys())}}, + take=len(uuid_to_unified), + ) + + found_uuids = {row.vector_store_id for row in rows} + + for uuid, original_id in uuid_to_unified.items(): + if uuid not in found_uuids: + raise HTTPException( + status_code=403, + detail=f"Vector store '{original_id}' not found or access denied.", + ) + + caller_team_id = user_api_key_dict.team_id + for row in rows: + vs_team_id = getattr(row, "team_id", None) + if vs_team_id is not None and vs_team_id != caller_team_id: + raise HTTPException( + status_code=403, + detail=( + f"Team '{caller_team_id}' does not have access to vector " + f"store '{row.vector_store_id}'. The store belongs to team " + f"'{vs_team_id}'." + ), + ) + async def get_model_file_id_mapping( self, file_ids: List[str], litellm_parent_otel_span: Span ) -> dict: @@ -954,7 +1048,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) else: file_object = await litellm.afile_retrieve( - custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai", + custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai", # type: ignore[arg-type] file_id=original_file_id, ) verbose_logger.debug( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index a5d6bc936bb..3d9a0df690d 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -536,6 +536,59 @@ def update_responses_input_with_model_file_ids( return updated_input +def _decode_vector_store_ids_in_tools( + tools: Optional[List[Dict[str, Any]]], +) -> Optional[List[Dict[str, Any]]]: + """ + Decodes unified (LiteLLM-managed) vector_store_ids in file_search tools to + provider-native IDs. Non-unified IDs are passed through unchanged. + + This runs unconditionally — no file-ID mapping is required. + """ + if not tools or not isinstance(tools, list): + return tools + + from litellm.llms.base_llm.managed_resources.utils import ( + is_base64_encoded_unified_id, + parse_unified_id, + ) + + updated_tools = [] + for tool in tools: + if not isinstance(tool, dict) or tool.get("type") != "file_search": + updated_tools.append(tool) + continue + + vector_store_ids = tool.get("vector_store_ids") + if not isinstance(vector_store_ids, list): + updated_tools.append(tool) + continue + + decoded_ids = [] + for vs_id in vector_store_ids: + if not isinstance(vs_id, str) or not is_base64_encoded_unified_id(vs_id): + decoded_ids.append(vs_id) + continue + + parsed = parse_unified_id(vs_id) + provider_resource_id = parsed.get("provider_resource_id") if parsed else None + + if not provider_resource_id: + verbose_logger.warning( + "file_search tool contains unified vector_store_id '%s' that could " + "not be decoded to a provider resource ID — passing original ID. " + "Ensure the vector store was created via LiteLLM.", + vs_id, + ) + decoded_ids.append(vs_id) + else: + decoded_ids.append(provider_resource_id) + + updated_tools.append({**tool, "vector_store_ids": decoded_ids}) + + return updated_tools + + def update_responses_tools_with_model_file_ids( tools: Optional[List[Dict[str, Any]]], model_id: Optional[str] = None, @@ -544,7 +597,8 @@ def update_responses_tools_with_model_file_ids( """ Updates responses API tools with provider-specific file IDs. - Handles code_interpreter tools with container.file_ids. + Pass 1 (always): decode unified vector_store_ids in file_search tools. + Pass 2 (needs mapping): map code_interpreter container file_ids to provider IDs. Args: tools: The responses API tools parameter @@ -555,6 +609,10 @@ def update_responses_tools_with_model_file_ids( if not tools or not isinstance(tools, list): return tools + # Pass 1: decode unified vector_store_ids (no mapping needed) + tools = _decode_vector_store_ids_in_tools(tools) or tools + + # Pass 2: map code_interpreter file IDs (requires mapping) if not model_file_id_mapping or not model_id: return tools diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index f429930e002..eea53fe06ec 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -54,6 +54,14 @@ class BaseResponsesAPIConfig(ABC): and v is not None } + def supports_native_file_search(self) -> bool: + """Return True if this provider handles the file_search tool natively. + + Override in provider subclasses that support file_search without + LiteLLM emulation (e.g. OpenAI, Azure OpenAI). + """ + return False + @abstractmethod def get_supported_openai_params(self, model: str) -> list: pass diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 9d909fd4017..cafb745862d 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -32,6 +32,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.OPENAI + def supports_native_file_search(self) -> bool: + return True + def get_supported_openai_params(self, model: str) -> list: """ All OpenAI Responses API params are supported diff --git a/litellm/responses/file_search/__init__.py b/litellm/responses/file_search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py new file mode 100644 index 00000000000..b50ed6f399b --- /dev/null +++ b/litellm/responses/file_search/emulated_handler.py @@ -0,0 +1,431 @@ +""" +Emulated file_search for providers that don't support the tool natively. + +Flow: + 1. Convert file_search tools to a single function tool definition. + 2. Call the provider with the function tool. + 3. If the provider issues a file_search function_call, execute vector search + via litellm.vector_stores.main.asearch(). + 4. Feed results back and get the final answer. + 5. Wrap everything in OpenAI Responses-API format: + [file_search_call output item] + [message output item with file_citation annotations] +""" + +import json +import time +import uuid +from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple, Union, cast + +import httpx + +from litellm._logging import verbose_logger +from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.vector_stores import VectorStoreSearchResult + +# Keep ToolParam broad so we stay compatible with both dict and Pydantic forms +ToolParam = Any + +FILE_SEARCH_FUNCTION_NAME = "litellm_file_search" + + +# --------------------------------------------------------------------------- +# Detection +# --------------------------------------------------------------------------- + +def should_use_emulated_file_search( + tools: Optional[Iterable[ToolParam]], + provider_config: Any, # BaseResponsesAPIConfig +) -> bool: + """Return True when there is a file_search tool and the provider can't handle it natively.""" + if not tools: + return False + has_fs = any( + isinstance(t, dict) and t.get("type") == "file_search" for t in tools + ) + if not has_fs: + return False + return provider_config is None or not provider_config.supports_native_file_search() + + +# --------------------------------------------------------------------------- +# Tool conversion +# --------------------------------------------------------------------------- + +def _build_function_tool(vector_store_ids: List[str]) -> Dict[str, Any]: + """ + Create an OpenAI function-tool definition that describes file search. + The function accepts a natural-language query; LiteLLM runs the actual + vector search against the configured vector stores. + """ + return { + "type": "function", + "function": { + "name": FILE_SEARCH_FUNCTION_NAME, + "description": ( + "Search the knowledge base for information relevant to the query. " + "Use this whenever you need to look up specific facts, documents, " + "or content from the vector store." + ), + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The search query to look up in the vector store.", + }, + "vector_store_id": { + "type": "string", + "description": "ID of the vector store to search.", + "enum": vector_store_ids, + }, + }, + "required": ["query"], + }, + }, + } + + +def _replace_file_search_tools( + tools: Optional[Iterable[ToolParam]], +) -> Tuple[List[Dict[str, Any]], List[str]]: + """ + Replace all file_search tools with a single function tool. + + Returns: + (new_tools_list, all_vector_store_ids) + """ + non_file_search: List[Dict[str, Any]] = [] + vector_store_ids: List[str] = [] + + for tool in (tools or []): + if isinstance(tool, dict) and tool.get("type") == "file_search": + ids = tool.get("vector_store_ids") or [] + vector_store_ids.extend(ids) + else: + non_file_search.append(tool) + + # Deduplicate while preserving order + unique_ids: List[str] = list(dict.fromkeys(vector_store_ids)) + if unique_ids: + non_file_search.append(_build_function_tool(unique_ids)) + + return non_file_search, unique_ids + + +# --------------------------------------------------------------------------- +# Search execution +# --------------------------------------------------------------------------- + +async def _run_vector_searches( + query: str, + vector_store_ids: List[str], + fallback_vector_store_ids: List[str], +) -> Tuple[List[str], List[VectorStoreSearchResult]]: + """ + Run `asearch` against all vector stores and collect results. + + Returns: + (queries_list, combined_results) + """ + import litellm.vector_stores.main as vs_main + + queries: List[str] = [query] + all_results: List[VectorStoreSearchResult] = [] + + ids_to_search = vector_store_ids or fallback_vector_store_ids + for vs_id in ids_to_search: + try: + response = await vs_main.asearch( + vector_store_id=vs_id, + query=query, + ) + results_data = response.get("data") if isinstance(response, dict) else getattr(response, "data", None) + if results_data: + all_results.extend(results_data) + except Exception as exc: + verbose_logger.warning( + "file_search emulated: search failed for vector_store_id='%s': %s", + vs_id, + exc, + ) + + return queries, all_results + + +# --------------------------------------------------------------------------- +# Result formatting +# --------------------------------------------------------------------------- + +def _format_search_results_as_tool_output( + results: List[VectorStoreSearchResult], +) -> str: + """Serialize search results into a string to pass back as the tool's output.""" + if not results: + return "No results found in the vector store." + + parts: List[str] = [] + for i, result in enumerate(results, 1): + score = getattr(result, "score", None) + file_id = getattr(result, "file_id", None) + filename = getattr(result, "filename", None) + content_items = getattr(result, "content", []) or [] + text_chunks = [ + c.get("text", "") if isinstance(c, dict) else getattr(c, "text", "") + for c in content_items + ] + text = " ".join(t for t in text_chunks if t) + + header = f"[Result {i}" + if filename: + header += f" | {filename}" + if file_id: + header += f" | file_id={file_id}" + if score is not None: + header += f" | score={score:.3f}" + header += "]" + + parts.append(f"{header}\n{text}") + + return "\n\n".join(parts) + + +def _build_file_search_call_output( + call_id: str, + queries: List[str], +) -> Dict[str, Any]: + """Build the file_search_call output item (mirrors OpenAI's format).""" + return { + "type": "file_search_call", + "id": call_id, + "status": "completed", + "queries": queries, + "search_results": None, + } + + +def _build_file_citation_annotations( + results: List[VectorStoreSearchResult], + text: str, +) -> List[Dict[str, Any]]: + """ + Build file_citation annotations for the text. + Each result with a file_id gets a citation at the end of the text. + """ + annotations: List[Dict[str, Any]] = [] + index = len(text) # cite at end of text block + seen_file_ids: set = set() + + for result in results: + file_id = getattr(result, "file_id", None) + filename = getattr(result, "filename", None) + if not file_id or file_id in seen_file_ids: + continue + seen_file_ids.add(file_id) + annotations.append( + { + "type": "file_citation", + "index": index, + "file_id": file_id, + "filename": filename or "", + } + ) + + return annotations + + +def _build_message_output( + response_text: str, + results: List[VectorStoreSearchResult], +) -> Dict[str, Any]: + """Build the message output item with optional file_citation annotations.""" + annotations = _build_file_citation_annotations(results, response_text) + return { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": response_text, + "annotations": annotations, + } + ], + } + + +def _extract_text_from_responses_output(response: ResponsesAPIResponse) -> str: + """Pull the assistant's text from the provider's response.""" + for item in response.output: + item_type = item.get("type") if isinstance(item, dict) else getattr(item, "type", None) + if item_type == "message": + content = item.get("content") if isinstance(item, dict) else getattr(item, "content", []) + for block in (content or []): + block_type = block.get("type") if isinstance(block, dict) else getattr(block, "type", None) + if block_type == "output_text": + raw = block.get("text") if isinstance(block, dict) else getattr(block, "text", "") + return str(raw) if raw is not None else "" + return "" + + +def _synthesize_responses_api_response( + original_response: ResponsesAPIResponse, + file_search_call_output: Dict[str, Any], + message_output: Dict[str, Any], +) -> ResponsesAPIResponse: + """ + Return a new ResponsesAPIResponse with: + output[0] = file_search_call item + output[1] = message item (with citations) + """ + import litellm + + return ResponsesAPIResponse( + id=getattr(original_response, "id", f"resp_{uuid.uuid4().hex}"), + object="response", + created_at=getattr(original_response, "created_at", int(time.time())), + status="completed", + model=getattr(original_response, "model", ""), + output=[file_search_call_output, message_output], + usage=getattr(original_response, "usage", None), + error=None, + ) + + +# --------------------------------------------------------------------------- +# Main entry point +# --------------------------------------------------------------------------- + +async def _call_aresponses(input, model, tools, **kwargs): # pragma: no cover – thin wrapper for patching in tests + from litellm.responses.main import aresponses + return await aresponses(input=input, model=model, tools=tools, **kwargs) + + +async def aresponses_with_emulated_file_search( + input: Any, + model: str, + tools: Optional[Iterable[ToolParam]] = None, + # Pass-through params — forwarded as-is to the underlying aresponses call + **kwargs: Any, +) -> ResponsesAPIResponse: + """ + Emulated file_search for providers that don't support it natively. + + Replaces file_search tools with a function tool, intercepts the tool call, + runs vector search, and synthesizes an OpenAI-format response. + """ + # 1. Replace file_search tools with function tool + transformed_tools, all_vs_ids = _replace_file_search_tools(tools) + + # 2. First provider call — provider will call the file_search function + first_response: ResponsesAPIResponse = cast( + ResponsesAPIResponse, + await _call_aresponses( + input=input, + model=model, + tools=transformed_tools or None, + **kwargs, + ), + ) + + # 3. Look for a file_search function_call in the output + file_search_calls = [ + item + for item in first_response.output + if ( + isinstance(item, dict) + and item.get("type") == "function_call" + and item.get("name") == FILE_SEARCH_FUNCTION_NAME + ) + or ( + hasattr(item, "type") + and getattr(item, "type") == "function_call" + and getattr(item, "name", None) == FILE_SEARCH_FUNCTION_NAME + ) + ] + + if not file_search_calls: + # Provider answered without calling the tool (e.g. it had enough context). + # Return as-is wrapped in OpenAI format. + call_id = f"fs_{uuid.uuid4().hex[:24]}" + response_text = _extract_text_from_responses_output(first_response) + return _synthesize_responses_api_response( + original_response=first_response, + file_search_call_output=_build_file_search_call_output(call_id, [str(input)]), + message_output=_build_message_output(response_text, []), + ) + + # 4. Execute each file_search tool call + tool_results: List[Dict[str, Any]] = [] + all_queries: List[str] = [] + all_results: List[VectorStoreSearchResult] = [] + file_search_call_id = f"fs_{uuid.uuid4().hex[:24]}" + + for tool_call in file_search_calls: + if isinstance(tool_call, dict): + call_id = tool_call.get("call_id") or tool_call.get("id") or file_search_call_id + raw_args = tool_call.get("arguments") or "{}" + else: + call_id = getattr(tool_call, "call_id", None) or getattr(tool_call, "id", file_search_call_id) + raw_args = getattr(tool_call, "arguments", "{}") or "{}" + + try: + args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args + except json.JSONDecodeError: + args = {} + + query = args.get("query", str(input)) + vs_id_arg = args.get("vector_store_id") + vs_ids_for_call = [vs_id_arg] if vs_id_arg else all_vs_ids + + queries, results = await _run_vector_searches( + query=query, + vector_store_ids=vs_ids_for_call, + fallback_vector_store_ids=all_vs_ids, + ) + all_queries.extend(queries) + all_results.extend(results) + + tool_results.append( + { + "type": "function_call_output", + "call_id": call_id, + "output": _format_search_results_as_tool_output(results), + } + ) + + # 5. Build follow-up input: original messages + assistant's tool call + tool results + original_input_items = list(input) if isinstance(input, (list, tuple)) else [{"role": "user", "content": str(input)}] + follow_up_input = ( + original_input_items + + [ + { + "type": "function_call", + "name": FILE_SEARCH_FUNCTION_NAME, + "call_id": file_search_calls[0].get("call_id") if isinstance(file_search_calls[0], dict) else getattr(file_search_calls[0], "call_id", file_search_call_id), + "arguments": file_search_calls[0].get("arguments") if isinstance(file_search_calls[0], dict) else getattr(file_search_calls[0], "arguments", "{}"), + } + ] + + tool_results + ) + + # 6. Follow-up call — provider writes the final answer given search results + final_response: ResponsesAPIResponse = cast( + ResponsesAPIResponse, + await _call_aresponses( + input=follow_up_input, + model=model, + tools=None, # no tools needed for the answer step + **{k: v for k, v in kwargs.items() if k not in ("tools",)}, + ), + ) + + # 7. Synthesize OpenAI-format output + response_text = _extract_text_from_responses_output(final_response) + + return _synthesize_responses_api_response( + original_response=final_response, + file_search_call_output=_build_file_search_call_output( + call_id=file_search_call_id, + queries=all_queries or [str(input)], + ), + message_output=_build_message_output(response_text, all_results), + ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index cd9ce67c26e..5438676c5fe 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -72,6 +72,15 @@ litellm_completion_transformation_handler = LiteLLMCompletionTransformationHandl ################################################# +def _has_file_search_tool(tools: Optional[Any]) -> bool: + """Return True if any tool in the list has type 'file_search'.""" + if not tools: + return False + return any( + isinstance(t, dict) and t.get("type") == "file_search" for t in tools + ) + + def mock_responses_api_response( mock_response: str = "In a peaceful grove beneath a silver moon, a unicorn named Lumina discovered a hidden pool that reflected the stars. As she dipped her horn into the water, the pool began to shimmer, revealing a pathway to a magical realm of endless night skies. Filled with wonder, Lumina whispered a wish for all who dream to find their own hidden magic, and as she glanced back, her hoofprints sparkled like stardust.", ): @@ -715,6 +724,50 @@ def responses( ) ) + if _has_file_search_tool(tools) and ( + responses_api_provider_config is None + or not responses_api_provider_config.supports_native_file_search() + ): + from litellm.responses.file_search.emulated_handler import ( + aresponses_with_emulated_file_search, + ) + + emulated_kwargs = { + "include": include, + "instructions": instructions, + "max_output_tokens": max_output_tokens, + "prompt": prompt, + "metadata": metadata, + "parallel_tool_calls": parallel_tool_calls, + "previous_response_id": previous_response_id, + "reasoning": reasoning, + "store": store, + "stream": stream, + "temperature": temperature, + "text": text, + "tool_choice": tool_choice, + "top_p": top_p, + "truncation": truncation, + "user": user, + "extra_headers": extra_headers, + "extra_query": extra_query, + "extra_body": extra_body, + "timeout": timeout, + "custom_llm_provider": custom_llm_provider, + **kwargs, + } + if _is_async: + return aresponses_with_emulated_file_search( + input=input, model=model, tools=tools, **emulated_kwargs + ) + return run_async_function( + aresponses_with_emulated_file_search, + input=input, + model=model, + tools=tools, + **emulated_kwargs, + ) + if responses_api_provider_config is None: return litellm_completion_transformation_handler.response_api_handler( model=model, diff --git a/tests/test_litellm/llms/test_file_search_responses.py b/tests/test_litellm/llms/test_file_search_responses.py new file mode 100644 index 00000000000..6f91b5386e6 --- /dev/null +++ b/tests/test_litellm/llms/test_file_search_responses.py @@ -0,0 +1,684 @@ +""" +Unit tests for Phase 1: file_search / vector_store support in the Responses API. + +Test plan reference: ~/.gstack/projects/BerriAI-litellm/sameerkankute-res-test-plan-*.md + +Coverage: + A1-A7 _decode_vector_store_ids_in_tools() + B1-B3 update_responses_tools_with_model_file_ids() + C1,D1 supports_native_file_search() + E1-E4 file_search guard in responses/main.py + F1-F6 ManagedFiles hook access control + G1-G3 get_vector_store_ids_from_file_search_tools() +""" + +import base64 +from typing import Any, Dict, List, Optional +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _decode_vector_store_ids_in_tools, + update_responses_tools_with_model_file_ids, +) +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_unified_vs_id( + unified_uuid: str = "abc-123", + provider_resource_id: str = "vs_provider_native", + model_id: str = "model-id-999", +) -> str: + """Build a valid base64-encoded unified vector-store ID.""" + raw = ( + f"litellm_proxy:vector_store;" + f"unified_id,{unified_uuid};" + f"model_id,{model_id};" + f"provider_resource_id,{provider_resource_id}" + ) + return base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=") + + +def _file_search_tool(vector_store_ids: Optional[List[str]] = None) -> Dict[str, Any]: + tool: Dict[str, Any] = {"type": "file_search"} + if vector_store_ids is not None: + tool["vector_store_ids"] = vector_store_ids + return tool + + +def _code_interpreter_tool(file_ids: Optional[List[str]] = None) -> Dict[str, Any]: + tool: Dict[str, Any] = {"type": "code_interpreter"} + if file_ids: + tool["container"] = {"type": "auto", "file_ids": file_ids} + return tool + + +# --------------------------------------------------------------------------- +# A-series: _decode_vector_store_ids_in_tools +# --------------------------------------------------------------------------- + +class TestDecodeVectorStoreIdsInTools: + def test_A1_none_input_returns_none(self): + assert _decode_vector_store_ids_in_tools(None) is None + + def test_A2_no_file_search_tools_unchanged(self): + tools = [{"type": "web_search"}, {"type": "code_interpreter"}] + result = _decode_vector_store_ids_in_tools(tools) + assert result == tools + + def test_A3_file_search_no_vector_store_ids_unchanged(self): + tools = [_file_search_tool()] # no vector_store_ids key + result = _decode_vector_store_ids_in_tools(tools) + assert result == tools + + def test_A4_unified_id_decoded_to_provider_resource_id(self): + unified_id = _make_unified_vs_id(provider_resource_id="vs_real_123") + tools = [_file_search_tool([unified_id])] + result = _decode_vector_store_ids_in_tools(tools) + assert result is not None + assert result[0]["vector_store_ids"] == ["vs_real_123"] + + def test_A5_native_id_passes_through_unchanged(self): + native_id = "vs_openai_abc" + tools = [_file_search_tool([native_id])] + result = _decode_vector_store_ids_in_tools(tools) + assert result is not None + assert result[0]["vector_store_ids"] == ["vs_openai_abc"] + + def test_A6_mixed_unified_and_native_ids(self): + unified_id = _make_unified_vs_id(provider_resource_id="vs_decoded") + native_id = "vs_native_xyz" + tools = [_file_search_tool([unified_id, native_id])] + result = _decode_vector_store_ids_in_tools(tools) + assert result is not None + assert result[0]["vector_store_ids"] == ["vs_decoded", "vs_native_xyz"] + + def test_A7_malformed_base64_passes_through_unchanged(self): + bad_id = "not_valid_base64!!!" + tools = [_file_search_tool([bad_id])] + result = _decode_vector_store_ids_in_tools(tools) + assert result is not None + assert result[0]["vector_store_ids"] == [bad_id] + + +# --------------------------------------------------------------------------- +# B-series: update_responses_tools_with_model_file_ids +# --------------------------------------------------------------------------- + +class TestUpdateResponsesToolsWithModelFileIds: + def test_B1_file_search_decode_runs_without_mapping(self): + """Decode pass executes even when model_file_id_mapping is None.""" + unified_id = _make_unified_vs_id(provider_resource_id="vs_decoded") + tools = [_file_search_tool([unified_id])] + + result = update_responses_tools_with_model_file_ids( + tools=tools, + model_id=None, + model_file_id_mapping=None, + ) + assert result is not None + assert result[0]["vector_store_ids"] == ["vs_decoded"] + + def test_B2_code_interpreter_mapping_still_works(self): + """code_interpreter mapping pass still works after decode pass.""" + model_id = "model-abc" + file_id = "litellm_managed_file_001" + tools = [_code_interpreter_tool([file_id])] + mapping = {file_id: {model_id: "provider_file_xyz"}} + + result = update_responses_tools_with_model_file_ids( + tools=tools, + model_id=model_id, + model_file_id_mapping=mapping, + ) + assert result is not None + assert result[0]["container"]["file_ids"] == ["provider_file_xyz"] + + def test_B3_both_passes_run_correctly(self): + """Both file_search decode and code_interpreter mapping run.""" + model_id = "model-abc" + file_id = "litellm_managed_file_001" + unified_id = _make_unified_vs_id(provider_resource_id="vs_decoded") + + tools = [ + _file_search_tool([unified_id]), + _code_interpreter_tool([file_id]), + ] + mapping = {file_id: {model_id: "provider_file_xyz"}} + + result = update_responses_tools_with_model_file_ids( + tools=tools, + model_id=model_id, + model_file_id_mapping=mapping, + ) + assert result is not None + assert result[0]["vector_store_ids"] == ["vs_decoded"] + assert result[1]["container"]["file_ids"] == ["provider_file_xyz"] + + +# --------------------------------------------------------------------------- +# C/D-series: supports_native_file_search +# --------------------------------------------------------------------------- + +class TestSupportsNativeFileSearch: + def test_C1_base_class_default_is_false(self): + # Access the unbound method directly — no need to instantiate an abstract class + assert BaseResponsesAPIConfig.supports_native_file_search(MagicMock()) is False + + def test_D1_openai_returns_true(self): + assert OpenAIResponsesAPIConfig().supports_native_file_search() is True + + +# --------------------------------------------------------------------------- +# E-series: file_search guard in responses/main.py +# --------------------------------------------------------------------------- + +class TestFileSearchGuardInResponsesMain: + """Tests for _has_file_search_tool helper and the UnsupportedParamsError guard.""" + + def test_has_file_search_tool_true(self): + from litellm.responses.main import _has_file_search_tool + + assert _has_file_search_tool([{"type": "file_search"}]) is True + + def test_has_file_search_tool_false_empty(self): + from litellm.responses.main import _has_file_search_tool + + assert _has_file_search_tool([]) is False + assert _has_file_search_tool(None) is False + + def test_has_file_search_tool_false_other_tools(self): + from litellm.responses.main import _has_file_search_tool + + assert _has_file_search_tool([{"type": "web_search"}]) is False + + def test_E1_openai_provider_no_error(self): + """OpenAI supports file_search natively — no error raised.""" + from litellm.llms.openai.responses.transformation import ( + OpenAIResponsesAPIConfig, + ) + from litellm.responses.main import _has_file_search_tool + + config = OpenAIResponsesAPIConfig() + tools = [{"type": "file_search", "vector_store_ids": ["vs_abc"]}] + assert _has_file_search_tool(tools) + assert config.supports_native_file_search() + # No exception expected — the guard would pass. + + def test_E2_no_provider_config_raises(self): + """Provider config is None → UnsupportedParamsError.""" + from litellm.exceptions import UnsupportedParamsError + from litellm.responses.main import _has_file_search_tool + + tools = [{"type": "file_search", "vector_store_ids": ["vs_abc"]}] + assert _has_file_search_tool(tools) + + with pytest.raises(UnsupportedParamsError): + if _has_file_search_tool(tools) and True: # config is None + raise UnsupportedParamsError( + message="Provider does not support file_search", + llm_provider="anthropic", + model="claude-3", + ) + + def test_E3_non_native_provider_config_raises(self): + """Provider config.supports_native_file_search() == False → error.""" + from litellm.exceptions import UnsupportedParamsError + from litellm.llms.base_llm.responses.transformation import ( + BaseResponsesAPIConfig, + ) + + mock_config = MagicMock(spec=BaseResponsesAPIConfig) + mock_config.supports_native_file_search.return_value = False + + tools = [{"type": "file_search"}] + with pytest.raises(UnsupportedParamsError): + if not mock_config.supports_native_file_search(): + raise UnsupportedParamsError( + message="Provider does not support file_search", + llm_provider="anthropic", + model="claude-3", + ) + + def test_E4_no_file_search_tools_no_error(self): + """No file_search tool in request → guard never fires.""" + from litellm.responses.main import _has_file_search_tool + + tools = [{"type": "web_search"}, {"type": "code_interpreter"}] + assert not _has_file_search_tool(tools) + + +# --------------------------------------------------------------------------- +# F-series: ManagedFiles hook — vector_store_ids access control +# --------------------------------------------------------------------------- + +class TestManagedFilesVectorStoreAccess: + def _make_hook(self): + """Return a ManagedFiles instance with prisma_client mocked.""" + from enterprise.litellm_enterprise.proxy.hooks.managed_files import ( + _PROXY_LiteLLMManagedFiles as ManagedFiles, + ) + + hook = ManagedFiles.__new__(ManagedFiles) + return hook + + def _make_user(self, team_id: Optional[str] = "team-abc") -> MagicMock: + user = MagicMock() + user.team_id = team_id + user.user_id = "user-1" + return user + + def test_F1_non_unified_vs_id_skipped(self): + hook = self._make_hook() + result = hook.get_vector_store_ids_from_file_search_tools( + [{"type": "file_search", "vector_store_ids": ["vs_native_123"]}] + ) + assert result == [] # native ID filtered out + + def test_F2_unified_vs_id_extracted(self): + hook = self._make_hook() + unified_id = _make_unified_vs_id() + result = hook.get_vector_store_ids_from_file_search_tools( + [{"type": "file_search", "vector_store_ids": [unified_id]}] + ) + assert result == [unified_id] + + @pytest.mark.asyncio + async def test_F3_wrong_team_raises_403(self): + from fastapi import HTTPException + + hook = self._make_hook() + unified_id = _make_unified_vs_id(unified_uuid="uuid-001") + + mock_row = MagicMock() + mock_row.vector_store_id = "uuid-001" + mock_row.team_id = "team-other" + + mock_db = MagicMock() + mock_db.litellm_managedvectorstorestable.find_many = AsyncMock( + return_value=[mock_row] + ) + + with patch( + "litellm.proxy.proxy_server.prisma_client", + MagicMock(db=mock_db), + ): + with pytest.raises(HTTPException) as exc_info: + await hook.check_vector_store_ids_access( + [unified_id], self._make_user(team_id="team-caller") + ) + assert exc_info.value.status_code == 403 + + @pytest.mark.asyncio + async def test_F4_no_team_on_vs_allowed(self): + """Legacy vector store with no team_id — accessible to all.""" + hook = self._make_hook() + unified_id = _make_unified_vs_id(unified_uuid="uuid-002") + + mock_row = MagicMock() + mock_row.vector_store_id = "uuid-002" + mock_row.team_id = None # legacy: no team restriction + + mock_db = MagicMock() + mock_db.litellm_managedvectorstorestable.find_many = AsyncMock( + return_value=[mock_row] + ) + + with patch( + "litellm.proxy.proxy_server.prisma_client", + MagicMock(db=mock_db), + ): + # Should not raise + await hook.check_vector_store_ids_access( + [unified_id], self._make_user(team_id="team-caller") + ) + + @pytest.mark.asyncio + async def test_F5_batch_lookup_single_db_call(self): + """Multiple unified IDs resolved in a single DB call (no N+1).""" + hook = self._make_hook() + ids = [ + _make_unified_vs_id(unified_uuid=f"uuid-{i}", provider_resource_id=f"vs_{i}") + for i in range(3) + ] + + rows = [] + for i in range(3): + r = MagicMock() + r.vector_store_id = f"uuid-{i}" + r.team_id = "team-abc" + rows.append(r) + + mock_db = MagicMock() + find_many_mock = AsyncMock(return_value=rows) + mock_db.litellm_managedvectorstorestable.find_many = find_many_mock + + with patch( + "litellm.proxy.proxy_server.prisma_client", + MagicMock(db=mock_db), + ): + await hook.check_vector_store_ids_access(ids, self._make_user("team-abc")) + + find_many_mock.assert_called_once() + + @pytest.mark.asyncio + async def test_F6_non_responses_call_type_skipped(self): + """Access check only runs for aresponses/responses call types.""" + from enterprise.litellm_enterprise.proxy.hooks.managed_files import ( + _PROXY_LiteLLMManagedFiles as ManagedFiles, + ) + from litellm.proxy._types import CallTypes + + # If call_type is acompletion, the vector_store check branch isn't reached. + # Smoke-test: hook runs without error for acompletion with file_search tools. + hook = MagicMock(spec=ManagedFiles) + hook.async_pre_call_hook = AsyncMock(return_value=None) + + await hook.async_pre_call_hook( + user_api_key_dict=self._make_user(), + cache=MagicMock(), + data={"tools": [{"type": "file_search", "vector_store_ids": ["vs_native"]}]}, + call_type=CallTypes.acompletion.value, + ) + hook.async_pre_call_hook.assert_called_once() + + +# --------------------------------------------------------------------------- +# G-series: get_vector_store_ids_from_file_search_tools helper +# --------------------------------------------------------------------------- + +class TestGetVectorStoreIdsFromFileSearchTools: + def _make_hook(self): + from enterprise.litellm_enterprise.proxy.hooks.managed_files import ( + _PROXY_LiteLLMManagedFiles as ManagedFiles, + ) + + return ManagedFiles.__new__(ManagedFiles) + + def test_G1_tools_none_returns_empty(self): + hook = self._make_hook() + assert hook.get_vector_store_ids_from_file_search_tools([]) == [] + + def test_G2_no_file_search_tools_returns_empty(self): + hook = self._make_hook() + tools = [{"type": "code_interpreter"}, {"type": "web_search"}] + assert hook.get_vector_store_ids_from_file_search_tools(tools) == [] + + def test_G3_only_file_search_vs_ids_returned(self): + hook = self._make_hook() + unified_id = _make_unified_vs_id() + tools = [ + {"type": "web_search"}, + {"type": "file_search", "vector_store_ids": [unified_id, "vs_native"]}, + {"type": "code_interpreter"}, + ] + result = hook.get_vector_store_ids_from_file_search_tools(tools) + # Only the unified ID is included; native IDs are filtered + assert result == [unified_id] + +# --------------------------------------------------------------------------- +# Phase 2: Emulated file_search handler +# --------------------------------------------------------------------------- + +class TestEmulatedFileSearchHandler: + """Tests for litellm/responses/file_search/emulated_handler.py""" + + def _make_mock_responses_api_response( + self, + text: str = "The answer is 42.", + output_type: str = "message", + include_function_call: bool = False, + ): + """Build a minimal ResponsesAPIResponse-like mock.""" + if include_function_call: + output = [ + { + "type": "function_call", + "name": "litellm_file_search", + "call_id": "call_abc123", + "arguments": '{"query": "what is X?", "vector_store_id": "vs_001"}', + } + ] + else: + output = [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": text}], + } + ] + resp = MagicMock() + resp.output = output + resp.id = "resp_test123" + resp.created_at = 1700000000 + resp.model = "claude-3-5-sonnet" + resp.usage = None + return resp + + # --- Tool conversion --- + + def test_H1_file_search_replaced_with_function_tool(self): + from litellm.responses.file_search.emulated_handler import ( + _replace_file_search_tools, + ) + + tools = [{"type": "file_search", "vector_store_ids": ["vs_abc", "vs_def"]}] + new_tools, vs_ids = _replace_file_search_tools(tools) + + assert vs_ids == ["vs_abc", "vs_def"] + assert len(new_tools) == 1 + assert new_tools[0]["type"] == "function" + assert new_tools[0]["function"]["name"] == "litellm_file_search" + # Both store IDs appear in the enum + enum_ids = new_tools[0]["function"]["parameters"]["properties"]["vector_store_id"]["enum"] + assert "vs_abc" in enum_ids + assert "vs_def" in enum_ids + + def test_H2_non_file_search_tools_preserved(self): + from litellm.responses.file_search.emulated_handler import ( + _replace_file_search_tools, + ) + + tools = [ + {"type": "web_search"}, + {"type": "file_search", "vector_store_ids": ["vs_abc"]}, + ] + new_tools, vs_ids = _replace_file_search_tools(tools) + + assert len(new_tools) == 2 # web_search + generated function tool + assert new_tools[0]["type"] == "web_search" + assert new_tools[1]["type"] == "function" + + def test_H3_no_file_search_tools_returns_unchanged(self): + from litellm.responses.file_search.emulated_handler import ( + _replace_file_search_tools, + ) + + tools = [{"type": "web_search"}] + new_tools, vs_ids = _replace_file_search_tools(tools) + + assert vs_ids == [] + assert new_tools == [{"type": "web_search"}] + + def test_H4_empty_vector_store_ids_no_function_tool(self): + from litellm.responses.file_search.emulated_handler import ( + _replace_file_search_tools, + ) + + tools = [{"type": "file_search", "vector_store_ids": []}] + new_tools, vs_ids = _replace_file_search_tools(tools) + + assert vs_ids == [] + assert new_tools == [] # no function tool added without store IDs + + # --- Detection --- + + def test_H5_should_use_emulated_for_non_native_provider(self): + from litellm.responses.file_search.emulated_handler import ( + should_use_emulated_file_search, + ) + + mock_config = MagicMock() + mock_config.supports_native_file_search.return_value = False + tools = [{"type": "file_search", "vector_store_ids": ["vs_abc"]}] + + assert should_use_emulated_file_search(tools, mock_config) is True + + def test_H6_should_not_emulate_for_native_provider(self): + from litellm.llms.openai.responses.transformation import ( + OpenAIResponsesAPIConfig, + ) + from litellm.responses.file_search.emulated_handler import ( + should_use_emulated_file_search, + ) + + config = OpenAIResponsesAPIConfig() + tools = [{"type": "file_search", "vector_store_ids": ["vs_abc"]}] + + assert should_use_emulated_file_search(tools, config) is False + + def test_H7_should_not_emulate_without_file_search_tools(self): + from litellm.responses.file_search.emulated_handler import ( + should_use_emulated_file_search, + ) + + mock_config = MagicMock() + mock_config.supports_native_file_search.return_value = False + tools = [{"type": "web_search"}] + + assert should_use_emulated_file_search(tools, mock_config) is False + + # --- Output synthesis --- + + def test_H8_synthesized_output_has_file_search_call_and_message(self): + from litellm.responses.file_search.emulated_handler import ( + _build_file_search_call_output, + _build_message_output, + ) + + fs_call = _build_file_search_call_output("fs_abc123", ["what is X?"]) + assert fs_call["type"] == "file_search_call" + assert fs_call["status"] == "completed" + assert fs_call["queries"] == ["what is X?"] + + msg = _build_message_output("The answer is 42.", []) + assert msg["type"] == "message" + assert msg["role"] == "assistant" + assert msg["content"][0]["type"] == "output_text" + assert msg["content"][0]["text"] == "The answer is 42." + + def test_H9_file_citations_added_for_results_with_file_ids(self): + from litellm.responses.file_search.emulated_handler import ( + _build_file_citation_annotations, + ) + + result = MagicMock() + result.file_id = "file-abc" + result.filename = "doc.pdf" + + annotations = _build_file_citation_annotations([result], "some text") + assert len(annotations) == 1 + assert annotations[0]["type"] == "file_citation" + assert annotations[0]["file_id"] == "file-abc" + assert annotations[0]["filename"] == "doc.pdf" + + def test_H10_no_duplicate_citations_for_same_file(self): + from litellm.responses.file_search.emulated_handler import ( + _build_file_citation_annotations, + ) + + r1, r2 = MagicMock(), MagicMock() + r1.file_id = "file-abc" + r1.filename = "doc.pdf" + r2.file_id = "file-abc" # same file + r2.filename = "doc.pdf" + + annotations = _build_file_citation_annotations([r1, r2], "text") + assert len(annotations) == 1 + + # --- End-to-end (mocked) --- + + @pytest.mark.asyncio + async def test_H11_emulated_full_flow_provider_calls_tool(self): + """Full flow: provider calls file_search function → search → follow-up → OpenAI output.""" + from litellm.responses.file_search.emulated_handler import ( + aresponses_with_emulated_file_search, + ) + + first_resp = self._make_mock_responses_api_response(include_function_call=True) + final_resp = self._make_mock_responses_api_response(text="Deep research enables multi-step queries.") + + search_result = MagicMock() + search_result.file_id = "file-xyz" + search_result.filename = "research.pdf" + search_result.score = 0.95 + search_result.content = [{"type": "text", "text": "deep research context..."}] + + mock_search_response = MagicMock() + mock_search_response.data = [search_result] + + with patch( + "litellm.responses.file_search.emulated_handler._call_aresponses", + new=AsyncMock(side_effect=[first_resp, final_resp]), + ), patch( + "litellm.vector_stores.main.asearch", + new=AsyncMock(return_value=mock_search_response), + ): + result = await aresponses_with_emulated_file_search( + input="What is deep research?", + model="anthropic/claude-3-5-sonnet", + tools=[{"type": "file_search", "vector_store_ids": ["vs_001"]}], + ) + + # output[0] is file_search_call, output[1] is message + # ResponsesAPIResponse converts dicts to Pydantic objects — use attribute access + def _get(item, key): + return item[key] if isinstance(item, dict) else getattr(item, key, None) + + assert _get(result.output[0], "type") == "file_search_call" + assert _get(result.output[0], "status") == "completed" + assert _get(result.output[1], "type") == "message" + content0 = _get(result.output[1], "content")[0] + assert "Deep research" in _get(content0, "text") + annotations = _get(content0, "annotations") + assert any(_get(a, "file_id") == "file-xyz" for a in annotations) + + @pytest.mark.asyncio + async def test_H12_emulated_flow_provider_answers_without_tool_call(self): + """If provider answers directly (no tool call), still return OpenAI format.""" + from litellm.responses.file_search.emulated_handler import ( + aresponses_with_emulated_file_search, + ) + + direct_resp = self._make_mock_responses_api_response(text="I already know the answer.") + + with patch( + "litellm.responses.file_search.emulated_handler._call_aresponses", + new=AsyncMock(return_value=direct_resp), + ): + result = await aresponses_with_emulated_file_search( + input="What is 2+2?", + model="anthropic/claude-3-5-sonnet", + tools=[{"type": "file_search", "vector_store_ids": ["vs_001"]}], + ) + + def _get(item, key): + return item[key] if isinstance(item, dict) else getattr(item, key, None) + + assert _get(result.output[0], "type") == "file_search_call" + assert _get(result.output[1], "type") == "message" + assert "I already know" in _get(_get(result.output[1], "content")[0], "text") + + def test_H13_should_use_emulated_when_provider_config_is_none(self): + """None provider config (chat fallback) also triggers emulation.""" + from litellm.responses.file_search.emulated_handler import ( + should_use_emulated_file_search, + ) + + tools = [{"type": "file_search", "vector_store_ids": ["vs_abc"]}] + assert should_use_emulated_file_search(tools, None) is True From 1d6c55de50c0aa6e6fce706632bc61a957bfa95e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 11:45:08 +0530 Subject: [PATCH 05/83] docs: add e2e testing tutorial for file_search Responses API Covers both paths: - Native passthrough (OpenAI/Azure): create vector store, run via SDK and proxy - Emulated fallback (Anthropic/any): register managed store, run via SDK and proxy Includes output format validation script and troubleshooting section. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../tutorials/file_search_responses_api.md | 325 ++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 docs/my-website/docs/tutorials/file_search_responses_api.md diff --git a/docs/my-website/docs/tutorials/file_search_responses_api.md b/docs/my-website/docs/tutorials/file_search_responses_api.md new file mode 100644 index 00000000000..9c8a773f202 --- /dev/null +++ b/docs/my-website/docs/tutorials/file_search_responses_api.md @@ -0,0 +1,325 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# File Search in the Responses API — E2E Testing Guide + +This tutorial walks you through end-to-end testing of the `file_search` tool in LiteLLM's Responses API. +Two paths are covered: + +| Path | When it runs | What LiteLLM does | +|---|---|---| +| **Native passthrough** | Provider natively supports `file_search` (OpenAI, Azure) | Decodes unified vector store ID → forwards to provider as-is | +| **Emulated fallback** | Provider doesn't support `file_search` (Anthropic, Bedrock, etc.) | Converts to a function tool → intercepts tool call → runs vector search → synthesizes OpenAI-format output | + +--- + +## Prerequisites + +```bash +pip install 'litellm[proxy]' +export OPENAI_API_KEY="sk-..." # for native path +export ANTHROPIC_API_KEY="sk-ant-..." # for emulated path +``` + +--- + +## Path 1: Native Passthrough (OpenAI) + +OpenAI natively handles `file_search`. LiteLLM decodes any unified vector store ID and forwards the request unchanged. + +### Step 1 — Create a vector store and upload a file + +```python +from openai import OpenAI + +client = OpenAI() # direct OpenAI call to set up test data + +# Upload a file +with open("knowledge.txt", "w") as f: + f.write("LiteLLM is a unified interface for 100+ LLM providers. " + "It supports chat completions, responses API, embeddings, and more.") + +file = client.files.create(file=open("knowledge.txt", "rb"), purpose="assistants") +print("file_id:", file.id) + +# Create a vector store and attach the file +vs = client.vector_stores.create(name="litellm-test-store") +client.vector_stores.files.create(vector_store_id=vs.id, file_id=file.id) +print("vector_store_id:", vs.id) +``` + +### Step 2 — Run file search via LiteLLM Python SDK + +```python showLineNumbers title="Native file_search via LiteLLM SDK" +import litellm + +response = litellm.responses( + model="openai/gpt-4.1", + input="What does LiteLLM support?", + tools=[{ + "type": "file_search", + "vector_store_ids": ["vs_abc123"] # replace with your vector_store_id + }], +) + +for item in response.output: + if item.type == "file_search_call": + print("Queries run:", item.queries) + print("Status:", item.status) + elif item.type == "message": + for block in item.content: + print("\nAnswer:", block.text) + for ann in block.annotations: + print(f" ↳ Citation: {ann.filename} (file_id={ann.file_id})") +``` + +**Expected output:** +``` +Queries run: ['What does LiteLLM support?'] +Status: completed + +Answer: LiteLLM is a unified interface for 100+ LLM providers... + ↳ Citation: knowledge.txt (file_id=file-xxxx) +``` + +### Step 3 — Run via LiteLLM Proxy + +Start the proxy: + +```bash title="config.yaml" +# config.yaml +model_list: + - model_name: gpt-4.1 + litellm_params: + model: openai/gpt-4.1 + api_key: os.environ/OPENAI_API_KEY +``` + +```bash +litellm --config config.yaml +``` + +Call the proxy: + +```python showLineNumbers title="Native file_search via LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:4000", api_key="any") + +response = client.responses.create( + model="gpt-4.1", + input="What does LiteLLM support?", + tools=[{"type": "file_search", "vector_store_ids": ["vs_abc123"]}], +) + +for item in response.output: + print(item.type, getattr(item, "queries", getattr(item, "content", ""))) +``` + +--- + +## Path 2: Emulated Fallback (Anthropic / any non-native provider) + +When you use a provider that doesn't natively support `file_search`, LiteLLM: +1. Converts the `file_search` tool to a function tool (`litellm_file_search`). +2. Lets the provider call the function with a natural-language query. +3. Runs your vector store search internally. +4. Feeds results back and makes a follow-up call. +5. Returns the final answer in OpenAI's `file_search_call` + `message` format. + +### Step 1 — Register a LiteLLM-managed vector store + +LiteLLM's vector store registry lets you configure any supported vector store backend (OpenAI, Pinecone, Milvus, Qdrant, etc.): + +```python showLineNumbers title="Register vector store via LiteLLM Proxy API" +import requests + +# Register the vector store with LiteLLM Proxy +resp = requests.post( + "http://localhost:4000/v1/vector_stores/new", + headers={"Authorization": "Bearer sk-your-proxy-key"}, + json={ + "vector_store_id": "my-openai-vs", # your logical name + "custom_llm_provider": "openai", + "vector_store_name": "litellm-test-store", + "litellm_params": { + "api_key": "sk-..." # provider API key (or use credentials in config.yaml) + }, + }, +) +print(resp.json()) +# Returns: {"vector_store_id": "bGl0ZWxsbV9wcm94eToB..."} ← LiteLLM unified ID +``` + +:::tip +Save the returned `vector_store_id` — this is the **LiteLLM-managed unified ID** that encodes the provider routing. Pass this in `vector_store_ids` and LiteLLM will decode it automatically. +::: + +### Step 2 — Run file search via LiteLLM SDK (emulated) + +```python showLineNumbers title="Emulated file_search with Anthropic" +import litellm + +# Use the unified vector_store_id returned by /v1/vector_stores/new +UNIFIED_VS_ID = "bGl0ZWxsbV9wcm94eToB..." + +response = litellm.responses( + model="anthropic/claude-sonnet-4-5", + input="What does LiteLLM support?", + tools=[{ + "type": "file_search", + "vector_store_ids": [UNIFIED_VS_ID] + }], +) + +for item in response.output: + if item.type == "file_search_call": + print("Queries run:", item.queries) + elif item.type == "message": + for block in item.content: + print("\nAnswer:", block.text) + for ann in block.annotations: + print(f" ↳ Citation: {ann.filename}") +``` + +LiteLLM automatically detects that Anthropic doesn't support `file_search` natively and routes through the emulated handler. + +### Step 3 — Run via LiteLLM Proxy (emulated) + +```bash title="config.yaml" +model_list: + - model_name: claude-sonnet + litellm_params: + model: anthropic/claude-sonnet-4-5 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +```python showLineNumbers title="Emulated file_search via LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:4000", api_key="sk-your-proxy-key") + +response = client.responses.create( + model="claude-sonnet", + input="What does LiteLLM support?", + tools=[{ + "type": "file_search", + "vector_store_ids": ["bGl0ZWxsbV9wcm94eToB..."] # unified ID + }], +) + +for item in response.output: + if hasattr(item, "type"): + if item.type == "file_search_call": + print("Queries:", item.queries) + elif item.type == "message": + print("Answer:", item.content[0].text) +``` + +--- + +## Validating the Output Format + +Regardless of which path ran, the response always follows the OpenAI Responses API format: + +```json +{ + "output": [ + { + "type": "file_search_call", + "id": "fs_abc123", + "status": "completed", + "queries": ["What does LiteLLM support?"], + "search_results": null + }, + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "LiteLLM is a unified interface...", + "annotations": [ + { + "type": "file_citation", + "index": 150, + "file_id": "file-xxxx", + "filename": "knowledge.txt" + } + ] + } + ] + } + ] +} +``` + +**Validation script:** + +```python showLineNumbers title="Validate response structure" +def validate_file_search_response(response): + """Assert that response follows OpenAI file_search output format.""" + output = response.output + assert len(output) >= 2, "Expected at least 2 output items" + + # First item: file_search_call + fs_call = output[0] + fs_type = fs_call["type"] if isinstance(fs_call, dict) else fs_call.type + assert fs_type == "file_search_call", f"Expected file_search_call, got {fs_type}" + + fs_status = fs_call["status"] if isinstance(fs_call, dict) else fs_call.status + assert fs_status == "completed" + + # Second item: message + msg = output[1] + msg_type = msg["type"] if isinstance(msg, dict) else msg.type + assert msg_type == "message" + + content = msg["content"] if isinstance(msg, dict) else msg.content + assert len(content) > 0 + text_block = content[0] + text = text_block["text"] if isinstance(text_block, dict) else text_block.text + assert isinstance(text, str) and len(text) > 0 + + print("✅ Response structure valid") + print(f" Queries: {fs_call['queries'] if isinstance(fs_call, dict) else fs_call.queries}") + print(f" Answer length: {len(text)} chars") + annotations = text_block["annotations"] if isinstance(text_block, dict) else text_block.annotations + print(f" Citations: {len(annotations)}") + +validate_file_search_response(response) +``` + +--- + +## Troubleshooting + +### `UnsupportedParamsError` is raised + +This means `file_search` was passed to a provider that doesn't support it natively, but the emulated fallback couldn't route either. Check: +- The model string is correct (e.g. `anthropic/claude-sonnet-4-5`, not just `claude-sonnet-4-5`) +- The `custom_llm_provider` is resolved — LiteLLM needs it to look up the provider config + +### Vector store search returns no results + +- Confirm the vector store ID exists and has files attached +- For LiteLLM-managed stores, ensure the file has finished processing (`status: completed`) +- Try a broader query string + +### `403 Access denied` on vector store + +The calling team doesn't have access to the vector store. Either: +- The vector store was created by a different team +- Use a proxy admin key to bypass team-scoped access control + +### Empty `annotations` in emulated mode + +The emulated path adds `file_citation` annotations only when the vector store search result includes a `file_id`. If your vector store provider doesn't return file-level metadata in search results, annotations will be empty — the answer text will still be populated. + +--- + +## What to check next + +- [File Search reference in Responses API docs](/docs/response_api#file-search-vector-stores) — full API reference +- [Vector Store management](/docs/vector_store_files) — create and manage vector stores +- [Managed vector stores](/docs/providers/bedrock_vector_store) — provider-specific setup From 289f698a3c2e6763a0f87b74ba89a90e2c153701 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 14:36:31 +0530 Subject: [PATCH 06/83] fix(responses): align emulated file_search output and multi-query behavior Ensure non-OpenAI emulated file_search matches native Responses output by populating search_results (when requested), fixing TypedDict field access, and supporting multi-query searches from tool calls. Made-with: Cursor --- .../responses/file_search/emulated_handler.py | 197 +++++++++++++----- 1 file changed, 144 insertions(+), 53 deletions(-) diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index b50ed6f399b..13a11cc5d1d 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -53,34 +53,43 @@ def should_use_emulated_file_search( def _build_function_tool(vector_store_ids: List[str]) -> Dict[str, Any]: """ - Create an OpenAI function-tool definition that describes file search. - The function accepts a natural-language query; LiteLLM runs the actual - vector search against the configured vector stores. + Create a Responses API function-tool definition that describes file search. + The function accepts one or more natural-language queries (like OpenAI's native + file_search); LiteLLM runs the actual vector search against the configured + vector stores. + + Note: Uses Responses API format (name/description/parameters at top level), + NOT Chat Completion format (nested under "function"), so that the + LiteLLMCompletionResponsesConfig transformation picks up name and description. """ return { "type": "function", - "function": { - "name": FILE_SEARCH_FUNCTION_NAME, - "description": ( - "Search the knowledge base for information relevant to the query. " - "Use this whenever you need to look up specific facts, documents, " - "or content from the vector store." - ), - "parameters": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "The search query to look up in the vector store.", - }, - "vector_store_id": { - "type": "string", - "description": "ID of the vector store to search.", - "enum": vector_store_ids, - }, + "name": FILE_SEARCH_FUNCTION_NAME, + "description": ( + "Search the knowledge base for information relevant to the query. " + "Use this whenever you need to look up specific facts, documents, " + "or content from the vector store. You can provide multiple queries " + "to search for different aspects of the information." + ), + "parameters": { + "type": "object", + "properties": { + "queries": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "One or more search queries to look up in the vector store. " + "Multiple queries help find comprehensive information from " + "different angles." + ), + }, + "vector_store_id": { + "type": "string", + "description": "ID of the vector store to search.", + "enum": vector_store_ids, }, - "required": ["query"], }, + "required": ["queries"], }, } @@ -117,37 +126,44 @@ def _replace_file_search_tools( # --------------------------------------------------------------------------- async def _run_vector_searches( - query: str, + queries: List[str], vector_store_ids: List[str], fallback_vector_store_ids: List[str], ) -> Tuple[List[str], List[VectorStoreSearchResult]]: """ - Run `asearch` against all vector stores and collect results. + Run `asearch` against all vector stores for all queries and collect results. + + Args: + queries: List of search queries to execute (like OpenAI's multi-query approach) + vector_store_ids: Specific vector store IDs to search + fallback_vector_store_ids: Fallback IDs if vector_store_ids is empty Returns: (queries_list, combined_results) """ import litellm.vector_stores.main as vs_main - queries: List[str] = [query] all_results: List[VectorStoreSearchResult] = [] - ids_to_search = vector_store_ids or fallback_vector_store_ids - for vs_id in ids_to_search: - try: - response = await vs_main.asearch( - vector_store_id=vs_id, - query=query, - ) - results_data = response.get("data") if isinstance(response, dict) else getattr(response, "data", None) - if results_data: - all_results.extend(results_data) - except Exception as exc: - verbose_logger.warning( - "file_search emulated: search failed for vector_store_id='%s': %s", - vs_id, - exc, - ) + + # Execute each query against all vector stores + for query in queries: + for vs_id in ids_to_search: + try: + response = await vs_main.asearch( + vector_store_id=vs_id, + query=query, + ) + results_data = response.get("data") if isinstance(response, dict) else getattr(response, "data", None) + if results_data: + all_results.extend(results_data) + except Exception as exc: + verbose_logger.warning( + "file_search emulated: search failed for query='%s', vector_store_id='%s': %s", + query, + vs_id, + exc, + ) return queries, all_results @@ -156,6 +172,13 @@ async def _run_vector_searches( # Result formatting # --------------------------------------------------------------------------- +def _get_field(result: Any, key: str, default: Any = None) -> Any: + """Read a field from either a dict/TypedDict or an attribute-based object.""" + if isinstance(result, dict): + return result.get(key, default) + return getattr(result, key, default) + + def _format_search_results_as_tool_output( results: List[VectorStoreSearchResult], ) -> str: @@ -165,10 +188,10 @@ def _format_search_results_as_tool_output( parts: List[str] = [] for i, result in enumerate(results, 1): - score = getattr(result, "score", None) - file_id = getattr(result, "file_id", None) - filename = getattr(result, "filename", None) - content_items = getattr(result, "content", []) or [] + score = _get_field(result, "score") + file_id = _get_field(result, "file_id") + filename = _get_field(result, "filename") + content_items = _get_field(result, "content") or [] text_chunks = [ c.get("text", "") if isinstance(c, dict) else getattr(c, "text", "") for c in content_items @@ -189,17 +212,57 @@ def _format_search_results_as_tool_output( return "\n\n".join(parts) +def _build_search_results_for_include( + results: List[VectorStoreSearchResult], +) -> List[Dict[str, Any]]: + """ + Convert VectorStoreSearchResult objects to the format expected in + file_search_call.search_results (mirrors OpenAI's include= format). + """ + formatted: List[Dict[str, Any]] = [] + for result in results: + content_items = _get_field(result, "content") or [] + text_chunks = [ + c.get("text", "") if isinstance(c, dict) else getattr(c, "text", "") + for c in content_items + ] + text = " ".join(t for t in text_chunks if t) + formatted.append( + { + "file_id": _get_field(result, "file_id") or "", + "filename": _get_field(result, "filename") or "", + "score": _get_field(result, "score"), + "text": text, + "attributes": _get_field(result, "attributes") or {}, + } + ) + return formatted + + def _build_file_search_call_output( call_id: str, queries: List[str], + results: Optional[List[VectorStoreSearchResult]] = None, + include_search_results: bool = False, ) -> Dict[str, Any]: - """Build the file_search_call output item (mirrors OpenAI's format).""" + """Build the file_search_call output item (mirrors OpenAI's format). + + Args: + call_id: Unique ID for this file_search call. + queries: List of search queries used. + results: The raw search results (used when include_search_results=True). + include_search_results: Populate search_results when the caller passed + ``include=["file_search_call.results"]``. + """ + search_results = None + if include_search_results and results: + search_results = _build_search_results_for_include(results) return { "type": "file_search_call", "id": call_id, "status": "completed", "queries": queries, - "search_results": None, + "search_results": search_results, } @@ -216,8 +279,8 @@ def _build_file_citation_annotations( seen_file_ids: set = set() for result in results: - file_id = getattr(result, "file_id", None) - filename = getattr(result, "filename", None) + file_id = _get_field(result, "file_id") + filename = _get_field(result, "filename") if not file_id or file_id in seen_file_ids: continue seen_file_ids.add(file_id) @@ -312,6 +375,19 @@ async def aresponses_with_emulated_file_search( Replaces file_search tools with a function tool, intercepts the tool call, runs vector search, and synthesizes an OpenAI-format response. """ + # Determine whether caller wants search_results populated in the output. + _include: List[str] = list(kwargs.get("include") or []) + _include_search_results = "file_search_call.results" in _include + + # Disable streaming for emulated file_search (not yet supported) + _original_stream = kwargs.get("stream") + if _original_stream: + verbose_logger.debug( + "Streaming is not yet supported for emulated file_search. " + "Disabling stream for this request." + ) + kwargs = {**kwargs, "stream": False} + # 1. Replace file_search tools with function tool transformed_tools, all_vs_ids = _replace_file_search_tools(tools) @@ -349,7 +425,12 @@ async def aresponses_with_emulated_file_search( response_text = _extract_text_from_responses_output(first_response) return _synthesize_responses_api_response( original_response=first_response, - file_search_call_output=_build_file_search_call_output(call_id, [str(input)]), + file_search_call_output=_build_file_search_call_output( + call_id=call_id, + queries=[str(input)], + results=None, + include_search_results=False, + ), message_output=_build_message_output(response_text, []), ) @@ -372,12 +453,20 @@ async def aresponses_with_emulated_file_search( except json.JSONDecodeError: args = {} - query = args.get("query", str(input)) + # Extract queries array (OpenAI-style multi-query support) + queries_from_call = args.get("queries") + if not queries_from_call: + # Fallback: check for single "query" field (backward compat) + single_query = args.get("query") + queries_from_call = [single_query] if single_query else [str(input)] + elif not isinstance(queries_from_call, list): + queries_from_call = [str(queries_from_call)] + vs_id_arg = args.get("vector_store_id") vs_ids_for_call = [vs_id_arg] if vs_id_arg else all_vs_ids queries, results = await _run_vector_searches( - query=query, + queries=queries_from_call, vector_store_ids=vs_ids_for_call, fallback_vector_store_ids=all_vs_ids, ) @@ -426,6 +515,8 @@ async def aresponses_with_emulated_file_search( file_search_call_output=_build_file_search_call_output( call_id=file_search_call_id, queries=all_queries or [str(input)], + results=all_results, + include_search_results=_include_search_results, ), message_output=_build_message_output(response_text, all_results), ) From e6d5e3af02cd838e8b173a44e90e00f53a3f7c73 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 14:36:38 +0530 Subject: [PATCH 07/83] fix(responses): avoid sending empty tools list in follow-up turns Drop tools=[] from transformed chat-completion requests so providers like Anthropic return normal assistant text after tool_result turns. Made-with: Cursor --- .../litellm_completion_transformation/transformation.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 71fa88fb751..3467dbbd27f 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -229,9 +229,13 @@ class LiteLLMCompletionResponsesConfig: if litellm_logging_obj: litellm_logging_obj.stream_options = stream_options - # only pass non-None values + # only pass non-None / non-empty values + # Explicitly exclude an empty tools list — sending tools=[] to providers + # like Anthropic in a tool_result conversation makes them return empty content. litellm_completion_request = { - k: v for k, v in litellm_completion_request.items() if v is not None + k: v + for k, v in litellm_completion_request.items() + if v is not None and not (k == "tools" and v == []) } return litellm_completion_request From 82c2dce6b994433b9630d9546c1bf8c51cf719f9 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 14:54:53 +0530 Subject: [PATCH 08/83] docs(file_search): streamline guide with usage tabs, architecture, and Q&A Replace duplicate path-by-path sections with a single usage-first doc format that includes SDK/Proxy tabs, an architecture diagram, and a focused Q&A section. Made-with: Cursor --- .../tutorials/file_search_responses_api.md | 351 +++++++----------- 1 file changed, 141 insertions(+), 210 deletions(-) diff --git a/docs/my-website/docs/tutorials/file_search_responses_api.md b/docs/my-website/docs/tutorials/file_search_responses_api.md index 9c8a773f202..3c642148df5 100644 --- a/docs/my-website/docs/tutorials/file_search_responses_api.md +++ b/docs/my-website/docs/tutorials/file_search_responses_api.md @@ -3,7 +3,17 @@ import TabItem from '@theme/TabItem'; # File Search in the Responses API — E2E Testing Guide -This tutorial walks you through end-to-end testing of the `file_search` tool in LiteLLM's Responses API. +LiteLLM now supports `file_search` in the Responses API across both: +- providers that support it natively (like OpenAI / Azure), and +- providers that do not (like Anthropic, Bedrock, and other non-native providers) via emulation. + +This page is both a quick blog-style overview and an end-to-end implementation guide. + +## What this is + +`file_search` lets models retrieve grounded context from your vector stores and answer with citations. +LiteLLM keeps one OpenAI-compatible output shape while routing requests through either native passthrough or an emulated fallback. + Two paths are covered: | Path | When it runs | What LiteLLM does | @@ -13,6 +23,117 @@ Two paths are covered: --- +## Usage + + + + +### 1. Setup `config.yaml` + +```yaml title="config.yaml" +model_list: + - model_name: gpt-4.1 + litellm_params: + model: openai/gpt-4.1 + api_key: os.environ/OPENAI_API_KEY + + - model_name: claude-sonnet + litellm_params: + model: anthropic/claude-sonnet-4-5 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +### 2. Start the proxy + +```bash +litellm --config config.yaml +``` + +### 3. Call Responses API with `file_search` + +```python title="Proxy call" +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:4000", api_key="sk-your-proxy-key") + +response = client.responses.create( + model="claude-sonnet", # swap to "gpt-4.1" for native path + input="What does LiteLLM support?", + tools=[{ + "type": "file_search", + "vector_store_ids": ["vs_abc123"] + }], + include=["file_search_call.results"], +) + +print(response.output) +``` + + + + +### 1. Install + set keys + +```bash +pip install litellm +export OPENAI_API_KEY="sk-..." +export ANTHROPIC_API_KEY="sk-ant-..." +``` + +### 2. Call Responses API with `file_search` + +```python title="SDK call" +import litellm + +response = litellm.responses( + model="anthropic/claude-sonnet-4-5", # swap to openai/gpt-4.1 for native path + input="What does LiteLLM support?", + tools=[{ + "type": "file_search", + "vector_store_ids": ["vs_abc123"] + }], + include=["file_search_call.results"], +) + +print(response.output) +``` + + + + +### Behavior Matrix + +| Path | SDK model | Proxy model | Behavior | +|---|---|---|---| +| Native passthrough | `openai/gpt-4.1` | `gpt-4.1` | Provider executes native `file_search` | +| Emulated fallback | `anthropic/claude-sonnet-4-5` | `claude-sonnet` | LiteLLM converts to function tool and synthesizes OpenAI-format output | + +--- + +## Architecture Diagram + +```mermaid +flowchart TD + A[Client SDK or Proxy Caller] --> B[LiteLLM Responses API] + B --> C{Provider supports native file_search?} + + C -->|Yes| D[Native passthrough path] + D --> D1[Decode unified vector_store_id if needed] + D1 --> D2[Forward request to provider unchanged] + D2 --> D3[Provider performs file_search] + D3 --> Z[OpenAI-compatible output] + + C -->|No| E[Emulated fallback path] + E --> E1[Convert file_search to litellm_file_search function tool] + E1 --> E2[First model call returns tool call with one or more queries] + E2 --> E3[LiteLLM executes vector search for each query] + E3 --> E4[Second model call with tool_result context] + E4 --> E5[Synthesize file_search_call + message + citations] + E5 --> Z[OpenAI-compatible output] +``` + +--- + ## Prerequisites ```bash @@ -23,200 +144,7 @@ export ANTHROPIC_API_KEY="sk-ant-..." # for emulated path --- -## Path 1: Native Passthrough (OpenAI) - -OpenAI natively handles `file_search`. LiteLLM decodes any unified vector store ID and forwards the request unchanged. - -### Step 1 — Create a vector store and upload a file - -```python -from openai import OpenAI - -client = OpenAI() # direct OpenAI call to set up test data - -# Upload a file -with open("knowledge.txt", "w") as f: - f.write("LiteLLM is a unified interface for 100+ LLM providers. " - "It supports chat completions, responses API, embeddings, and more.") - -file = client.files.create(file=open("knowledge.txt", "rb"), purpose="assistants") -print("file_id:", file.id) - -# Create a vector store and attach the file -vs = client.vector_stores.create(name="litellm-test-store") -client.vector_stores.files.create(vector_store_id=vs.id, file_id=file.id) -print("vector_store_id:", vs.id) -``` - -### Step 2 — Run file search via LiteLLM Python SDK - -```python showLineNumbers title="Native file_search via LiteLLM SDK" -import litellm - -response = litellm.responses( - model="openai/gpt-4.1", - input="What does LiteLLM support?", - tools=[{ - "type": "file_search", - "vector_store_ids": ["vs_abc123"] # replace with your vector_store_id - }], -) - -for item in response.output: - if item.type == "file_search_call": - print("Queries run:", item.queries) - print("Status:", item.status) - elif item.type == "message": - for block in item.content: - print("\nAnswer:", block.text) - for ann in block.annotations: - print(f" ↳ Citation: {ann.filename} (file_id={ann.file_id})") -``` - -**Expected output:** -``` -Queries run: ['What does LiteLLM support?'] -Status: completed - -Answer: LiteLLM is a unified interface for 100+ LLM providers... - ↳ Citation: knowledge.txt (file_id=file-xxxx) -``` - -### Step 3 — Run via LiteLLM Proxy - -Start the proxy: - -```bash title="config.yaml" -# config.yaml -model_list: - - model_name: gpt-4.1 - litellm_params: - model: openai/gpt-4.1 - api_key: os.environ/OPENAI_API_KEY -``` - -```bash -litellm --config config.yaml -``` - -Call the proxy: - -```python showLineNumbers title="Native file_search via LiteLLM Proxy" -from openai import OpenAI - -client = OpenAI(base_url="http://localhost:4000", api_key="any") - -response = client.responses.create( - model="gpt-4.1", - input="What does LiteLLM support?", - tools=[{"type": "file_search", "vector_store_ids": ["vs_abc123"]}], -) - -for item in response.output: - print(item.type, getattr(item, "queries", getattr(item, "content", ""))) -``` - ---- - -## Path 2: Emulated Fallback (Anthropic / any non-native provider) - -When you use a provider that doesn't natively support `file_search`, LiteLLM: -1. Converts the `file_search` tool to a function tool (`litellm_file_search`). -2. Lets the provider call the function with a natural-language query. -3. Runs your vector store search internally. -4. Feeds results back and makes a follow-up call. -5. Returns the final answer in OpenAI's `file_search_call` + `message` format. - -### Step 1 — Register a LiteLLM-managed vector store - -LiteLLM's vector store registry lets you configure any supported vector store backend (OpenAI, Pinecone, Milvus, Qdrant, etc.): - -```python showLineNumbers title="Register vector store via LiteLLM Proxy API" -import requests - -# Register the vector store with LiteLLM Proxy -resp = requests.post( - "http://localhost:4000/v1/vector_stores/new", - headers={"Authorization": "Bearer sk-your-proxy-key"}, - json={ - "vector_store_id": "my-openai-vs", # your logical name - "custom_llm_provider": "openai", - "vector_store_name": "litellm-test-store", - "litellm_params": { - "api_key": "sk-..." # provider API key (or use credentials in config.yaml) - }, - }, -) -print(resp.json()) -# Returns: {"vector_store_id": "bGl0ZWxsbV9wcm94eToB..."} ← LiteLLM unified ID -``` - -:::tip -Save the returned `vector_store_id` — this is the **LiteLLM-managed unified ID** that encodes the provider routing. Pass this in `vector_store_ids` and LiteLLM will decode it automatically. -::: - -### Step 2 — Run file search via LiteLLM SDK (emulated) - -```python showLineNumbers title="Emulated file_search with Anthropic" -import litellm - -# Use the unified vector_store_id returned by /v1/vector_stores/new -UNIFIED_VS_ID = "bGl0ZWxsbV9wcm94eToB..." - -response = litellm.responses( - model="anthropic/claude-sonnet-4-5", - input="What does LiteLLM support?", - tools=[{ - "type": "file_search", - "vector_store_ids": [UNIFIED_VS_ID] - }], -) - -for item in response.output: - if item.type == "file_search_call": - print("Queries run:", item.queries) - elif item.type == "message": - for block in item.content: - print("\nAnswer:", block.text) - for ann in block.annotations: - print(f" ↳ Citation: {ann.filename}") -``` - -LiteLLM automatically detects that Anthropic doesn't support `file_search` natively and routes through the emulated handler. - -### Step 3 — Run via LiteLLM Proxy (emulated) - -```bash title="config.yaml" -model_list: - - model_name: claude-sonnet - litellm_params: - model: anthropic/claude-sonnet-4-5 - api_key: os.environ/ANTHROPIC_API_KEY -``` - -```python showLineNumbers title="Emulated file_search via LiteLLM Proxy" -from openai import OpenAI - -client = OpenAI(base_url="http://localhost:4000", api_key="sk-your-proxy-key") - -response = client.responses.create( - model="claude-sonnet", - input="What does LiteLLM support?", - tools=[{ - "type": "file_search", - "vector_store_ids": ["bGl0ZWxsbV9wcm94eToB..."] # unified ID - }], -) - -for item in response.output: - if hasattr(item, "type"): - if item.type == "file_search_call": - print("Queries:", item.queries) - elif item.type == "message": - print("Answer:", item.content[0].text) -``` - ---- +## Example response shape ## Validating the Output Format @@ -292,29 +220,32 @@ validate_file_search_response(response) --- -## Troubleshooting +## Q&A -### `UnsupportedParamsError` is raised +### Q: Why do I see `UnsupportedParamsError`? -This means `file_search` was passed to a provider that doesn't support it natively, but the emulated fallback couldn't route either. Check: -- The model string is correct (e.g. `anthropic/claude-sonnet-4-5`, not just `claude-sonnet-4-5`) -- The `custom_llm_provider` is resolved — LiteLLM needs it to look up the provider config +A: This usually means `file_search` was passed to a provider that does not support it natively and emulation could not route correctly. +Check: +- The model string is valid (for example, `anthropic/claude-sonnet-4-5`). +- `custom_llm_provider` resolves correctly so LiteLLM can load the provider config. -### Vector store search returns no results +### Q: Why does vector search return no results? -- Confirm the vector store ID exists and has files attached -- For LiteLLM-managed stores, ensure the file has finished processing (`status: completed`) -- Try a broader query string +A: Common causes: +- The vector store ID is wrong or has no files attached. +- In LiteLLM-managed stores, file ingestion is not complete (`status != completed`). +- The query is too narrow; try a broader query. -### `403 Access denied` on vector store +### Q: Why am I getting `403 Access denied` on vector store calls? -The calling team doesn't have access to the vector store. Either: -- The vector store was created by a different team -- Use a proxy admin key to bypass team-scoped access control +A: The caller does not have access to that vector store. +- The store may belong to another team. +- Use an admin/proxy key if your setup requires cross-team access. -### Empty `annotations` in emulated mode +### Q: Why are `annotations` empty in emulated mode? -The emulated path adds `file_citation` annotations only when the vector store search result includes a `file_id`. If your vector store provider doesn't return file-level metadata in search results, annotations will be empty — the answer text will still be populated. +A: `file_citation` annotations require `file_id` metadata in search results. +If your vector backend does not return file-level metadata, the answer text is still generated but citations can be empty. --- From e22d9031e0a50ea522fa4d549bd1e8e1e9e88028 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 14:59:55 +0530 Subject: [PATCH 09/83] docs(response_api): move file_search details to dedicated tutorial Replace inline file_search documentation in response_api.md with a canonical link and add the new tutorial to sidebars so users discover the usage-first guide. Made-with: Cursor --- docs/my-website/docs/response_api.md | 127 +-------------------------- docs/my-website/sidebars.js | 1 + 2 files changed, 3 insertions(+), 125 deletions(-) diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index 183b3399009..3df84faa668 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -1558,132 +1558,9 @@ curl -X POST "http://localhost:4000/v1/responses" \ ## File Search (Vector Stores) -The **file_search** tool lets the model search your vector stores and cite retrieved content in its answer (OpenAI Responses API format). Pass `tools=[{"type": "file_search", "vector_store_ids": [...]}]`. The response includes a `file_search_call` output item and `file_citation` annotations on the answer text. +For full `file_search` usage (native + emulated fallback), SDK/Proxy examples, architecture diagram, and Q&A, see: -**Supported providers:** `openai`, `azure` (native). Other providers will receive an `UnsupportedParamsError` until the emulated-fallback path is available. - -:::note -If you are using LiteLLM-managed vector stores (created via `/v1/vector_stores`), pass the LiteLLM vector store ID directly — LiteLLM automatically decodes it to the provider-native ID before sending the request. -::: - -### Python SDK - -```python showLineNumbers title="File search with LiteLLM Python SDK" -import litellm - -response = litellm.responses( - model="openai/gpt-4.1", - input="What is deep research?", - tools=[{ - "type": "file_search", - "vector_store_ids": ["vs_abc123"] # native or LiteLLM-managed vector store ID - }], -) - -# Output contains a file_search_call item followed by the answer with citations -for item in response.output: - if item.type == "file_search_call": - print("Queries:", item.queries) - elif item.type == "message": - for block in item.content: - print(block.text) - for ann in block.annotations: - print(f" ↳ {ann.filename} (file_id={ann.file_id})") -``` - -#### Response Format - -```json -{ - "output": [ - { - "type": "file_search_call", - "id": "fs_67c09ccea8c48191ade9367e3ba71515", - "status": "completed", - "queries": ["What is deep research?"], - "search_results": null - }, - { - "id": "msg_67c09cd3091c819185af2be5d13d87de", - "type": "message", - "role": "assistant", - "content": [ - { - "type": "output_text", - "text": "Deep research is a capability that allows for extensive inquiry ...", - "annotations": [ - { - "type": "file_citation", - "index": 992, - "file_id": "file-2dtbBZdjtDKS8eqWxqbgDi", - "filename": "deep_research_blog.pdf" - } - ] - } - ] - } - ] -} -``` - -### LiteLLM Proxy (AI Gateway) - -**OpenAI Python SDK (proxy as base_url):** - -```python showLineNumbers title="File search via LiteLLM Proxy" -from openai import OpenAI - -client = OpenAI( - base_url="http://localhost:4000", - api_key="your-proxy-api-key", -) - -response = client.responses.create( - model="openai/gpt-4.1", - input="Summarise the Q3 earnings report.", - tools=[{ - "type": "file_search", - "vector_store_ids": ["vs_abc123"] - }], -) -``` - -**curl:** - -```bash title="File search via curl to LiteLLM Proxy" -curl -X POST "http://localhost:4000/v1/responses" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer your-proxy-api-key" \ - -d '{ - "model": "openai/gpt-4.1", - "input": "Summarise the Q3 earnings report.", - "tools": [{"type": "file_search", "vector_store_ids": ["vs_abc123"]}] - }' -``` - -### Using LiteLLM-Managed Vector Stores - -If you created a vector store through LiteLLM (`POST /v1/vector_stores/new`), use the returned `vector_store_id` directly. LiteLLM decodes the unified ID to the provider-native vector store ID automatically. - -```python showLineNumbers title="File search with LiteLLM-managed vector store" -from openai import OpenAI - -client = OpenAI(base_url="http://localhost:4000", api_key="your-proxy-api-key") - -# vector_store_id returned by POST /v1/vector_stores/new -managed_vs_id = "bGl0ZWxsbV9wcm94eTo..." # LiteLLM-managed ID - -response = client.responses.create( - model="openai/gpt-4.1", - input="What does the documentation say about authentication?", - tools=[{"type": "file_search", "vector_store_ids": [managed_vs_id]}], -) -``` - -LiteLLM will: -1. Verify the calling team has access to the vector store. -2. Decode the managed ID to the provider-native vector store ID. -3. Forward the request to the provider unchanged. +- [`File Search in the Responses API — E2E Testing Guide`](/docs/tutorials/file_search_responses_api) ## Session Management diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 1362745a91f..4a8d67409ad 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -1054,6 +1054,7 @@ const sidebars = { label: "AI Coding Tools (OpenWebUI, Claude Code, Gemini CLI, OpenAI Codex, etc.)", href: "/docs/ai_tools", }, + "tutorials/file_search_responses_api", "tutorials/anthropic_file_usage", "tutorials/default_team_self_serve", "tutorials/msft_sso", From 729f7d48eb0f575bc11b0e13b870f96fe522aaf5 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 15:10:46 +0530 Subject: [PATCH 10/83] fix(file_search): address greptile review on follow-up calls and tests Include all function_call items when building emulated follow-up input and update tests to assert real emulated routing + Responses-format function tool structure. Made-with: Cursor --- .../responses/file_search/emulated_handler.py | 24 ++-- .../llms/test_file_search_responses.py | 110 +++++++++++++----- 2 files changed, 101 insertions(+), 33 deletions(-) diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 13a11cc5d1d..2ac1d3dea67 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -481,18 +481,28 @@ async def aresponses_with_emulated_file_search( } ) - # 5. Build follow-up input: original messages + assistant's tool call + tool results + # 5. Build follow-up input: original messages + all assistant tool calls + tool results original_input_items = list(input) if isinstance(input, (list, tuple)) else [{"role": "user", "content": str(input)}] - follow_up_input = ( - original_input_items - + [ + follow_up_function_calls: List[Dict[str, Any]] = [] + for tc in file_search_calls: + if isinstance(tc, dict): + tc_call_id = tc.get("call_id") or tc.get("id") or file_search_call_id + tc_args = tc.get("arguments") or "{}" + else: + tc_call_id = getattr(tc, "call_id", None) or getattr(tc, "id", file_search_call_id) + tc_args = getattr(tc, "arguments", "{}") or "{}" + follow_up_function_calls.append( { "type": "function_call", "name": FILE_SEARCH_FUNCTION_NAME, - "call_id": file_search_calls[0].get("call_id") if isinstance(file_search_calls[0], dict) else getattr(file_search_calls[0], "call_id", file_search_call_id), - "arguments": file_search_calls[0].get("arguments") if isinstance(file_search_calls[0], dict) else getattr(file_search_calls[0], "arguments", "{}"), + "call_id": tc_call_id, + "arguments": tc_args, } - ] + ) + + follow_up_input = ( + original_input_items + + follow_up_function_calls + tool_results ) diff --git a/tests/test_litellm/llms/test_file_search_responses.py b/tests/test_litellm/llms/test_file_search_responses.py index 6f91b5386e6..f1eb2650542 100644 --- a/tests/test_litellm/llms/test_file_search_responses.py +++ b/tests/test_litellm/llms/test_file_search_responses.py @@ -179,7 +179,7 @@ class TestSupportsNativeFileSearch: # --------------------------------------------------------------------------- class TestFileSearchGuardInResponsesMain: - """Tests for _has_file_search_tool helper and the UnsupportedParamsError guard.""" + """Tests for _has_file_search_tool helper and emulated routing guard.""" def test_has_file_search_tool_true(self): from litellm.responses.main import _has_file_search_tool @@ -210,40 +210,98 @@ class TestFileSearchGuardInResponsesMain: assert config.supports_native_file_search() # No exception expected — the guard would pass. - def test_E2_no_provider_config_raises(self): - """Provider config is None → UnsupportedParamsError.""" - from litellm.exceptions import UnsupportedParamsError - from litellm.responses.main import _has_file_search_tool + def test_E2_no_provider_config_routes_to_emulated_handler(self): + """Provider config None + file_search should route to emulated handler.""" + from litellm.responses.main import responses tools = [{"type": "file_search", "vector_store_ids": ["vs_abc"]}] - assert _has_file_search_tool(tools) + logging_obj = MagicMock() + expected = {"ok": True} - with pytest.raises(UnsupportedParamsError): - if _has_file_search_tool(tools) and True: # config is None - raise UnsupportedParamsError( - message="Provider does not support file_search", - llm_provider="anthropic", - model="claude-3", - ) + with ( + patch( + "litellm.responses.main.litellm.get_llm_provider", + return_value=("claude-sonnet-4-5", "anthropic", None, None), + ), + patch( + "litellm.responses.main.update_responses_input_with_model_file_ids", + return_value="hello", + ), + patch( + "litellm.responses.main.update_responses_tools_with_model_file_ids", + return_value=tools, + ), + patch( + "litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config", + return_value=None, + ), + patch( + "litellm.responses.main.ResponsesAPIRequestUtils.get_requested_response_api_optional_param", + return_value={}, + ), + patch("litellm.responses.main.run_async_function", return_value=expected) as run_async_mock, + ): + result = responses( + input="hello", + model="anthropic/claude-sonnet-4-5", + tools=tools, + litellm_logging_obj=logging_obj, + litellm_call_id="call-123", + ) - def test_E3_non_native_provider_config_raises(self): - """Provider config.supports_native_file_search() == False → error.""" - from litellm.exceptions import UnsupportedParamsError + assert result == expected + assert run_async_mock.called + routed_func = run_async_mock.call_args.args[0] + assert routed_func.__name__ == "aresponses_with_emulated_file_search" + + def test_E3_non_native_provider_config_routes_to_emulated_handler(self): + """Non-native provider config + file_search should route to emulated handler.""" from litellm.llms.base_llm.responses.transformation import ( BaseResponsesAPIConfig, ) + from litellm.responses.main import responses + tools = [{"type": "file_search", "vector_store_ids": ["vs_abc"]}] + logging_obj = MagicMock() + expected = {"ok": True} mock_config = MagicMock(spec=BaseResponsesAPIConfig) mock_config.supports_native_file_search.return_value = False - tools = [{"type": "file_search"}] - with pytest.raises(UnsupportedParamsError): - if not mock_config.supports_native_file_search(): - raise UnsupportedParamsError( - message="Provider does not support file_search", - llm_provider="anthropic", - model="claude-3", - ) + with ( + patch( + "litellm.responses.main.litellm.get_llm_provider", + return_value=("claude-sonnet-4-5", "anthropic", None, None), + ), + patch( + "litellm.responses.main.update_responses_input_with_model_file_ids", + return_value="hello", + ), + patch( + "litellm.responses.main.update_responses_tools_with_model_file_ids", + return_value=tools, + ), + patch( + "litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config", + return_value=mock_config, + ), + patch( + "litellm.responses.main.ResponsesAPIRequestUtils.get_requested_response_api_optional_param", + return_value={}, + ), + patch("litellm.responses.main.run_async_function", return_value=expected) as run_async_mock, + ): + result = responses( + input="hello", + model="anthropic/claude-sonnet-4-5", + tools=tools, + litellm_logging_obj=logging_obj, + litellm_call_id="call-123", + ) + + assert result == expected + assert run_async_mock.called + routed_func = run_async_mock.call_args.args[0] + assert routed_func.__name__ == "aresponses_with_emulated_file_search" def test_E4_no_file_search_tools_no_error(self): """No file_search tool in request → guard never fires.""" @@ -473,9 +531,9 @@ class TestEmulatedFileSearchHandler: assert vs_ids == ["vs_abc", "vs_def"] assert len(new_tools) == 1 assert new_tools[0]["type"] == "function" - assert new_tools[0]["function"]["name"] == "litellm_file_search" + assert new_tools[0]["name"] == "litellm_file_search" # Both store IDs appear in the enum - enum_ids = new_tools[0]["function"]["parameters"]["properties"]["vector_store_id"]["enum"] + enum_ids = new_tools[0]["parameters"]["properties"]["vector_store_id"]["enum"] assert "vs_abc" in enum_ids assert "vs_def" in enum_ids From 77a5093ce287db8bdd32ca753a2dd84977cd066e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 15:20:56 +0530 Subject: [PATCH 11/83] fix(file_search): preserve emulated response params and hidden metadata Forward explicit responses() params on emulated file search calls and preserve hidden params on synthesized responses so callback billing/logging context is retained. Made-with: Cursor --- litellm/responses/file_search/emulated_handler.py | 9 +++++---- litellm/responses/main.py | 5 +++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 2ac1d3dea67..85a8f789988 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -339,9 +339,7 @@ def _synthesize_responses_api_response( output[0] = file_search_call item output[1] = message item (with citations) """ - import litellm - - return ResponsesAPIResponse( + synthesized = ResponsesAPIResponse( id=getattr(original_response, "id", f"resp_{uuid.uuid4().hex}"), object="response", created_at=getattr(original_response, "created_at", int(time.time())), @@ -351,6 +349,9 @@ def _synthesize_responses_api_response( usage=getattr(original_response, "usage", None), error=None, ) + if hasattr(original_response, "_hidden_params"): + synthesized._hidden_params = getattr(original_response, "_hidden_params") + return synthesized # --------------------------------------------------------------------------- @@ -513,7 +514,7 @@ async def aresponses_with_emulated_file_search( input=follow_up_input, model=model, tools=None, # no tools needed for the answer step - **{k: v for k, v in kwargs.items() if k not in ("tools",)}, + **kwargs, ), ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 5438676c5fe..761be5a2e14 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -742,6 +742,7 @@ def responses( "previous_response_id": previous_response_id, "reasoning": reasoning, "store": store, + "background": background, "stream": stream, "temperature": temperature, "text": text, @@ -749,6 +750,10 @@ def responses( "top_p": top_p, "truncation": truncation, "user": user, + "service_tier": service_tier, + "safety_identifier": safety_identifier, + "text_format": text_format, + "allowed_openai_params": allowed_openai_params, "extra_headers": extra_headers, "extra_query": extra_query, "extra_body": extra_body, From 5692db812389efd118a14fb403b1da1e278c56ee Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 15:33:11 +0530 Subject: [PATCH 12/83] fix(file_search): address latest greptile feedback Strip internal logging ids from emulated sub-calls, dedupe included search_results by file_id, clean unused imports, and add unit coverage for dedupe behavior. Made-with: Cursor --- .../responses/file_search/emulated_handler.py | 12 +++++++---- litellm/responses/main.py | 3 ++- .../llms/test_file_search_responses.py | 21 +++++++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 85a8f789988..5d2c23fdfa3 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -14,9 +14,7 @@ Flow: import json import time import uuid -from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple, Union, cast - -import httpx +from typing import Any, Dict, Iterable, List, Optional, Tuple, cast from litellm._logging import verbose_logger from litellm.types.llms.openai import ResponsesAPIResponse @@ -220,7 +218,13 @@ def _build_search_results_for_include( file_search_call.search_results (mirrors OpenAI's include= format). """ formatted: List[Dict[str, Any]] = [] + seen_file_ids: set = set() for result in results: + file_id = _get_field(result, "file_id") or "" + if file_id and file_id in seen_file_ids: + continue + if file_id: + seen_file_ids.add(file_id) content_items = _get_field(result, "content") or [] text_chunks = [ c.get("text", "") if isinstance(c, dict) else getattr(c, "text", "") @@ -229,7 +233,7 @@ def _build_search_results_for_include( text = " ".join(t for t in text_chunks if t) formatted.append( { - "file_id": _get_field(result, "file_id") or "", + "file_id": file_id, "filename": _get_field(result, "filename") or "", "score": _get_field(result, "score"), "text": text, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 761be5a2e14..4404e6b366a 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -732,6 +732,7 @@ def responses( aresponses_with_emulated_file_search, ) + _internal_skip = {"litellm_logging_obj", "litellm_call_id", "aresponses"} emulated_kwargs = { "include": include, "instructions": instructions, @@ -759,7 +760,7 @@ def responses( "extra_body": extra_body, "timeout": timeout, "custom_llm_provider": custom_llm_provider, - **kwargs, + **{k: v for k, v in kwargs.items() if k not in _internal_skip}, } if _is_async: return aresponses_with_emulated_file_search( diff --git a/tests/test_litellm/llms/test_file_search_responses.py b/tests/test_litellm/llms/test_file_search_responses.py index f1eb2650542..63599781c17 100644 --- a/tests/test_litellm/llms/test_file_search_responses.py +++ b/tests/test_litellm/llms/test_file_search_responses.py @@ -659,6 +659,27 @@ class TestEmulatedFileSearchHandler: annotations = _build_file_citation_annotations([r1, r2], "text") assert len(annotations) == 1 + def test_H14_include_search_results_dedupes_by_file_id(self): + from litellm.responses.file_search.emulated_handler import ( + _build_search_results_for_include, + ) + + r1, r2 = MagicMock(), MagicMock() + r1.file_id = "file-abc" + r1.filename = "doc.pdf" + r1.score = 0.9 + r1.attributes = {} + r1.content = [{"type": "text", "text": "first hit"}] + r2.file_id = "file-abc" # same file appears for a second query + r2.filename = "doc.pdf" + r2.score = 0.85 + r2.attributes = {} + r2.content = [{"type": "text", "text": "second hit"}] + + search_results = _build_search_results_for_include([r1, r2]) + assert len(search_results) == 1 + assert search_results[0]["file_id"] == "file-abc" + # --- End-to-end (mocked) --- @pytest.mark.asyncio From 8eb8756e844049dd9d1f9b32d1f8f27058e28d0f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 15:55:41 +0530 Subject: [PATCH 13/83] fix: Preserve annotations in Azure AI Foundry Agents responses Azure AI Agents with Grounding (e.g., Bing Search) include annotations (citation URLs) in responses, but the handler was dropping them during transformation. This fix: - Extracts annotations from text content in agent responses - Transforms them to OpenAI-compatible ChatCompletionAnnotation format - Passes annotations through all completion paths (sync, async, streaming) - Handles both polling and SSE streaming responses Fixes #19126 Co-Authored-By: Claude Haiku 4.5 --- litellm/llms/azure_ai/agents/handler.py | 110 +++++++++++++++---- tests/llm_translation/test_azure_agents.py | 117 ++++++++++++++++++++- 2 files changed, 207 insertions(+), 20 deletions(-) diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index 9eeec7f4e36..5b779acb0d1 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -97,14 +97,63 @@ class AzureAIAgentsHandler: # ------------------------------------------------------------------------- # Response Helpers # ------------------------------------------------------------------------- - def _extract_content_from_messages(self, messages_data: dict) -> str: - """Extract assistant content from the messages response.""" + def _extract_content_from_messages( + self, messages_data: dict + ) -> Tuple[str, Optional[List[Dict[str, Any]]]]: + """Extract assistant content and annotations from the messages response. + + Returns (content, annotations) where annotations is a list of + OpenAI-compatible ChatCompletionAnnotation dicts, or None. + """ for msg in messages_data.get("data", []): if msg.get("role") == "assistant": for content_item in msg.get("content", []): if content_item.get("type") == "text": - return content_item.get("text", {}).get("value", "") - return "" + text_obj = content_item.get("text", {}) + content = text_obj.get("value", "") + raw_annotations = text_obj.get("annotations") + annotations = self._transform_annotations( + raw_annotations + ) + return content, annotations + return "", None + + def _transform_annotations( + self, + raw_annotations: Optional[List[Dict[str, Any]]], + ) -> Optional[List[Dict[str, Any]]]: + """Transform Azure AI Foundry annotations to OpenAI-compatible format. + + Azure AI returns annotations like: + {"type": "url_citation", "text": "[1]", "start_index": 10, + "end_index": 13, "url_citation": {"url": "...", "title": "..."}} + + OpenAI expects: + {"type": "url_citation", "url_citation": {"url": "...", "title": "...", + "start_index": 10, "end_index": 13}} + """ + if not raw_annotations: + return None + + result: List[Dict[str, Any]] = [] + for ann in raw_annotations: + ann_type = ann.get("type", "url_citation") + if ann_type == "url_citation": + url_citation = dict(ann.get("url_citation", {})) + # Azure puts start/end_index at annotation level; OpenAI + # expects them inside url_citation + if "start_index" in ann and "start_index" not in url_citation: + url_citation["start_index"] = ann["start_index"] + if "end_index" in ann and "end_index" not in url_citation: + url_citation["end_index"] = ann["end_index"] + result.append( + {"type": "url_citation", "url_citation": url_citation} + ) + else: + # Pass through unknown annotation types as-is + result.append(ann) + + return result if result else None def _build_model_response( self, @@ -113,15 +162,23 @@ class AzureAIAgentsHandler: model_response: ModelResponse, thread_id: str, messages: List[Dict[str, Any]], + annotations: Optional[List[Dict[str, Any]]] = None, ) -> ModelResponse: """Build the ModelResponse from agent output.""" from litellm.types.utils import Choices, Message, Usage + message_kwargs: Dict[str, Any] = { + "content": content, + "role": "assistant", + } + if annotations: + message_kwargs["annotations"] = annotations + model_response.choices = [ Choices( finish_reason="stop", index=0, - message=Message(content=content, role="assistant"), + message=Message(**message_kwargs), ) ] model_response.model = model @@ -250,7 +307,7 @@ class AzureAIAgentsHandler: ) # Execute the agent flow - thread_id, content = self._execute_agent_flow_sync( + thread_id, content, annotations = self._execute_agent_flow_sync( make_request=make_request, api_base=api_base, api_version=api_version, @@ -261,7 +318,7 @@ class AzureAIAgentsHandler: ) return self._build_model_response( - model, content, model_response, thread_id, messages + model, content, model_response, thread_id, messages, annotations ) def _execute_agent_flow_sync( @@ -273,8 +330,8 @@ class AzureAIAgentsHandler: thread_id: Optional[str], messages: List[Dict[str, Any]], optional_params: dict, - ) -> Tuple[str, str]: - """Execute the agent flow synchronously. Returns (thread_id, content).""" + ) -> Tuple[str, str, Optional[List[Dict[str, Any]]]]: + """Execute the agent flow synchronously. Returns (thread_id, content, annotations).""" # Step 1: Create thread if not provided if not thread_id: @@ -347,8 +404,8 @@ class AzureAIAgentsHandler: ) self._check_response(response, [200], "Failed to get messages") - content = self._extract_content_from_messages(response.json()) - return thread_id, content + content, annotations = self._extract_content_from_messages(response.json()) + return thread_id, content, annotations # ------------------------------------------------------------------------- # Async Completion @@ -399,7 +456,7 @@ class AzureAIAgentsHandler: ) # Execute the agent flow - thread_id, content = await self._execute_agent_flow_async( + thread_id, content, annotations = await self._execute_agent_flow_async( make_request=make_request, api_base=api_base, api_version=api_version, @@ -410,7 +467,7 @@ class AzureAIAgentsHandler: ) return self._build_model_response( - model, content, model_response, thread_id, messages + model, content, model_response, thread_id, messages, annotations ) async def _execute_agent_flow_async( @@ -422,8 +479,8 @@ class AzureAIAgentsHandler: thread_id: Optional[str], messages: List[Dict[str, Any]], optional_params: dict, - ) -> Tuple[str, str]: - """Execute the agent flow asynchronously. Returns (thread_id, content).""" + ) -> Tuple[str, str, Optional[List[Dict[str, Any]]]]: + """Execute the agent flow asynchronously. Returns (thread_id, content, annotations).""" # Step 1: Create thread if not provided if not thread_id: @@ -496,8 +553,8 @@ class AzureAIAgentsHandler: ) self._check_response(response, [200], "Failed to get messages") - content = self._extract_content_from_messages(response.json()) - return thread_id, content + content, annotations = self._extract_content_from_messages(response.json()) + return thread_id, content, annotations # ------------------------------------------------------------------------- # Streaming Completion (Native SSE) @@ -585,6 +642,7 @@ class AzureAIAgentsHandler: response_id = f"chatcmpl-{uuid.uuid4().hex[:8]}" created = int(time.time()) thread_id = None + collected_annotations: Optional[List[Dict[str, Any]]] = None current_event = None @@ -600,6 +658,9 @@ class AzureAIAgentsHandler: if data_str == "[DONE]": # Send final chunk with finish_reason + final_delta_kwargs: Dict[str, Any] = {"content": None} + if collected_annotations: + final_delta_kwargs["annotations"] = collected_annotations final_chunk = ModelResponseStream( id=response_id, created=created, @@ -609,7 +670,7 @@ class AzureAIAgentsHandler: StreamingChoices( finish_reason="stop", index=0, - delta=Delta(content=None), + delta=Delta(**final_delta_kwargs), ) ], ) @@ -628,6 +689,19 @@ class AzureAIAgentsHandler: thread_id = data["id"] verbose_logger.debug(f"Stream created thread: {thread_id}") + # Extract annotations from completed message + if current_event == "thread.message.completed": + for content_item in data.get("content", []): + if content_item.get("type") == "text": + raw_annotations = content_item.get("text", {}).get( + "annotations" + ) + transformed = self._transform_annotations( + raw_annotations + ) + if transformed: + collected_annotations = transformed + # Process message deltas - this is where the actual content comes if current_event == "thread.message.delta": delta_content = data.get("delta", {}).get("content", []) diff --git a/tests/llm_translation/test_azure_agents.py b/tests/llm_translation/test_azure_agents.py index 66a46d53383..19ce49a3bc9 100644 --- a/tests/llm_translation/test_azure_agents.py +++ b/tests/llm_translation/test_azure_agents.py @@ -343,13 +343,126 @@ def test_azure_ai_agents_extract_content_from_messages(): ] } - content = handler._extract_content_from_messages(messages_data) + content, annotations = handler._extract_content_from_messages(messages_data) assert content == "The answer is 100." + assert annotations is None # Test empty response empty_data = {"data": []} - content = handler._extract_content_from_messages(empty_data) + content, annotations = handler._extract_content_from_messages(empty_data) assert content == "" + assert annotations is None + + +def test_azure_ai_agents_extract_content_with_annotations(): + """ + Test that annotations (e.g., Bing Search citations) are extracted from + Azure Agents message responses and transformed to OpenAI-compatible format. + + Ref: https://github.com/BerriAI/litellm/issues/19126 + """ + from litellm.llms.azure_ai.agents.handler import AzureAIAgentsHandler + + handler = AzureAIAgentsHandler() + + messages_data = { + "data": [ + { + "id": "msg_abc", + "role": "assistant", + "content": [ + { + "type": "text", + "text": { + "value": "According to sources [1], the answer is yes.", + "annotations": [ + { + "type": "url_citation", + "text": "[1]", + "start_index": 22, + "end_index": 25, + "url_citation": { + "url": "https://example.com/source", + "title": "Example Source" + } + } + ] + } + } + ] + } + ] + } + + content, annotations = handler._extract_content_from_messages(messages_data) + assert content == "According to sources [1], the answer is yes." + assert annotations is not None + assert len(annotations) == 1 + assert annotations[0]["type"] == "url_citation" + assert annotations[0]["url_citation"]["url"] == "https://example.com/source" + assert annotations[0]["url_citation"]["title"] == "Example Source" + # start/end_index should be moved into url_citation for OpenAI compatibility + assert annotations[0]["url_citation"]["start_index"] == 22 + assert annotations[0]["url_citation"]["end_index"] == 25 + + +def test_azure_ai_agents_build_model_response_with_annotations(): + """ + Test that _build_model_response includes annotations in the Message object. + """ + from litellm.llms.azure_ai.agents.handler import AzureAIAgentsHandler + from litellm.types.utils import ModelResponse + + handler = AzureAIAgentsHandler() + model_response = ModelResponse() + + annotations = [ + { + "type": "url_citation", + "url_citation": { + "url": "https://example.com", + "title": "Example", + "start_index": 0, + "end_index": 5, + }, + } + ] + + result = handler._build_model_response( + model="azure_ai/agents/asst_123", + content="Hello [1]", + model_response=model_response, + thread_id="thread_abc", + messages=[{"role": "user", "content": "test"}], + annotations=annotations, + ) + + assert result.choices[0].message.content == "Hello [1]" + assert result.choices[0].message.annotations is not None + assert len(result.choices[0].message.annotations) == 1 + assert result.choices[0].message.annotations[0]["type"] == "url_citation" + + +def test_azure_ai_agents_build_model_response_without_annotations(): + """ + Test that _build_model_response works correctly without annotations. + """ + from litellm.llms.azure_ai.agents.handler import AzureAIAgentsHandler + from litellm.types.utils import ModelResponse + + handler = AzureAIAgentsHandler() + model_response = ModelResponse() + + result = handler._build_model_response( + model="azure_ai/agents/asst_123", + content="Hello", + model_response=model_response, + thread_id="thread_abc", + messages=[{"role": "user", "content": "test"}], + ) + + assert result.choices[0].message.content == "Hello" + assert getattr(result.choices[0].message, "annotations", None) is None @pytest.mark.asyncio From a286050293b448bdf8eb3d33755df9ce6b2915d9 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 17:28:36 +0530 Subject: [PATCH 14/83] Add basic gpt-5.4 mini and nano entry in model map --- ...odel_prices_and_context_window_backup.json | 68 +++++++++++++++++++ model_prices_and_context_window.json | 68 +++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6786fc33595..81b5c690bea 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18747,6 +18747,74 @@ "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true }, + "gpt-5.4-mini": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "openai", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true + }, + "gpt-5.4-nano": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.25e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_none_reasoning_effort": false, + "supports_xhigh_reasoning_effort": true + }, "gpt-5-pro": { "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6786fc33595..81b5c690bea 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18747,6 +18747,74 @@ "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true }, + "gpt-5.4-mini": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "openai", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true + }, + "gpt-5.4-nano": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.25e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_none_reasoning_effort": false, + "supports_xhigh_reasoning_effort": true + }, "gpt-5-pro": { "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, From 8a8047e5190fc7c3c0bbca692a19bef2a184eadc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 17:32:58 +0530 Subject: [PATCH 15/83] Add all missing entries in model entries --- ...odel_prices_and_context_window_backup.json | 30 +++++++++++++++++++ model_prices_and_context_window.json | 30 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 81b5c690bea..4a627d4aa16 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18749,13 +18749,27 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1.5e-07, + "cache_read_input_token_cost_flex": 3.9e-08, + "cache_read_input_token_cost_priority": 1.5e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 3e-07, "input_cost_per_token": 7.5e-07, + "input_cost_per_token_above_272k_tokens": 1.5e-06, + "input_cost_per_token_flex": 3.75e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_priority": 1.5e-06, + "input_cost_per_token_above_272k_tokens_priority": 3e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, + "output_cost_per_token_above_272k_tokens": 6.75e-06, + "output_cost_per_token_flex": 2.25e-06, + "output_cost_per_token_batches": 2.25e-06, + "output_cost_per_token_priority": 6.75e-06, + "output_cost_per_token_above_272k_tokens_priority": 1.0125e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18777,19 +18791,34 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_service_tier": true, "supports_vision": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true }, "gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4e-08, + "cache_read_input_token_cost_flex": 1.04e-08, + "cache_read_input_token_cost_priority": 4e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "input_cost_per_token": 2e-07, + "input_cost_per_token_above_272k_tokens": 4e-07, + "input_cost_per_token_flex": 1e-07, + "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_priority": 4e-07, + "input_cost_per_token_above_272k_tokens_priority": 8e-07, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.25e-06, + "output_cost_per_token_above_272k_tokens": 1.875e-06, + "output_cost_per_token_flex": 6.25e-07, + "output_cost_per_token_batches": 6.25e-07, + "output_cost_per_token_priority": 1.875e-06, + "output_cost_per_token_above_272k_tokens_priority": 2.8125e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18811,6 +18840,7 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_service_tier": true, "supports_vision": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 81b5c690bea..4a627d4aa16 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18749,13 +18749,27 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1.5e-07, + "cache_read_input_token_cost_flex": 3.9e-08, + "cache_read_input_token_cost_priority": 1.5e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 3e-07, "input_cost_per_token": 7.5e-07, + "input_cost_per_token_above_272k_tokens": 1.5e-06, + "input_cost_per_token_flex": 3.75e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_priority": 1.5e-06, + "input_cost_per_token_above_272k_tokens_priority": 3e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, + "output_cost_per_token_above_272k_tokens": 6.75e-06, + "output_cost_per_token_flex": 2.25e-06, + "output_cost_per_token_batches": 2.25e-06, + "output_cost_per_token_priority": 6.75e-06, + "output_cost_per_token_above_272k_tokens_priority": 1.0125e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18777,19 +18791,34 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_service_tier": true, "supports_vision": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true }, "gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4e-08, + "cache_read_input_token_cost_flex": 1.04e-08, + "cache_read_input_token_cost_priority": 4e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "input_cost_per_token": 2e-07, + "input_cost_per_token_above_272k_tokens": 4e-07, + "input_cost_per_token_flex": 1e-07, + "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_priority": 4e-07, + "input_cost_per_token_above_272k_tokens_priority": 8e-07, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.25e-06, + "output_cost_per_token_above_272k_tokens": 1.875e-06, + "output_cost_per_token_flex": 6.25e-07, + "output_cost_per_token_batches": 6.25e-07, + "output_cost_per_token_priority": 1.875e-06, + "output_cost_per_token_above_272k_tokens_priority": 2.8125e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18811,6 +18840,7 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_service_tier": true, "supports_vision": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true From 34af653ff8b050a914166a96543c2646e3f8dd7f Mon Sep 17 00:00:00 2001 From: Milan Date: Tue, 17 Mar 2026 14:20:16 +0200 Subject: [PATCH 16/83] docs: note min version for encrypted_content_affinity Document that encrypted_content_affinity requires LiteLLM >= 1.82.1 to prevent /responses invalid_encrypted_content when routing shifts deployments. Made-with: Cursor --- docs/my-website/docs/proxy/config_settings.md | 2 +- docs/my-website/docs/proxy/load_balancing.md | 2 +- docs/my-website/docs/response_api.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index a0e404e3a18..1d6fc1b03bd 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -361,7 +361,7 @@ router_settings: | redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** | | cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. | | router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) | -| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity`, `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` | +| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity` (**requires LiteLLM >= 1.82.1**), `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` | | deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). | | ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. | | search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) | diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 5bf39d179f6..313df99b259 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -352,7 +352,7 @@ If `order=1` deployment is unavailable (e.g., rate-limited), the router falls ba When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or organizations), encrypted content items (like `rs_...` reasoning items) can only be decrypted by the originating API key. -**Solution:** Use the `encrypted_content_affinity` pre-call check to automatically route follow-up requests containing encrypted items to the correct deployment: +**Solution:** Use the `encrypted_content_affinity` pre-call check (**requires LiteLLM >= 1.82.1**) to automatically route follow-up requests containing encrypted items to the correct deployment: ```yaml model_list: diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index fb55ae9f9d0..66aa2e1ad95 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -1160,7 +1160,7 @@ follow_up = await router.aresponses( To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks` in your proxy config.yaml. - `responses_api_deployment_check`: high priority routing when `previous_response_id` is provided -- `encrypted_content_affinity`: **[Recommended]** content-aware routing for encrypted items (e.g., `rs_...` reasoning items) +- `encrypted_content_affinity`: **[Recommended]** content-aware routing for encrypted items (e.g., `rs_...` reasoning items) (**requires LiteLLM >= 1.82.1**) - `session_affinity`: sticky sessions based on session id (takes priority over `deployment_affinity`) - `deployment_affinity`: sticky sessions based on user key (applies even without `previous_response_id`) From 464ac7be12e14f61c0b5b677e7f631be0c95890a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 18:08:07 +0530 Subject: [PATCH 17/83] Fix doc --- docs/my-website/docs/tutorials/file_search_responses_api.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/my-website/docs/tutorials/file_search_responses_api.md b/docs/my-website/docs/tutorials/file_search_responses_api.md index 3c642148df5..5ff2adee0aa 100644 --- a/docs/my-website/docs/tutorials/file_search_responses_api.md +++ b/docs/my-website/docs/tutorials/file_search_responses_api.md @@ -1,14 +1,12 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# File Search in the Responses API — E2E Testing Guide +# File Search in the Responses API LiteLLM now supports `file_search` in the Responses API across both: - providers that support it natively (like OpenAI / Azure), and - providers that do not (like Anthropic, Bedrock, and other non-native providers) via emulation. -This page is both a quick blog-style overview and an end-to-end implementation guide. - ## What this is `file_search` lets models retrieve grounded context from your vector stores and answer with citations. From 8b7eac5dc93db889a43f7ef233e44472b832b447 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 18:10:24 +0530 Subject: [PATCH 18/83] Fix doc --- .../docs/tutorials/file_search_responses_api.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/my-website/docs/tutorials/file_search_responses_api.md b/docs/my-website/docs/tutorials/file_search_responses_api.md index 5ff2adee0aa..9df18b4095e 100644 --- a/docs/my-website/docs/tutorials/file_search_responses_api.md +++ b/docs/my-website/docs/tutorials/file_search_responses_api.md @@ -15,11 +15,11 @@ LiteLLM keeps one OpenAI-compatible output shape while routing requests through Two paths are covered: | Path | When it runs | What LiteLLM does | -|---|---|---| +|||| | **Native passthrough** | Provider natively supports `file_search` (OpenAI, Azure) | Decodes unified vector store ID → forwards to provider as-is | | **Emulated fallback** | Provider doesn't support `file_search` (Anthropic, Bedrock, etc.) | Converts to a function tool → intercepts tool call → runs vector search → synthesizes OpenAI-format output | ---- + ## Usage @@ -102,11 +102,11 @@ print(response.output) ### Behavior Matrix | Path | SDK model | Proxy model | Behavior | -|---|---|---|---| +||||| | Native passthrough | `openai/gpt-4.1` | `gpt-4.1` | Provider executes native `file_search` | | Emulated fallback | `anthropic/claude-sonnet-4-5` | `claude-sonnet` | LiteLLM converts to function tool and synthesizes OpenAI-format output | ---- + ## Architecture Diagram @@ -130,7 +130,7 @@ flowchart TD E5 --> Z[OpenAI-compatible output] ``` ---- + ## Prerequisites @@ -140,7 +140,7 @@ export OPENAI_API_KEY="sk-..." # for native path export ANTHROPIC_API_KEY="sk-ant-..." # for emulated path ``` ---- + ## Example response shape @@ -216,7 +216,7 @@ def validate_file_search_response(response): validate_file_search_response(response) ``` ---- + ## Q&A @@ -245,7 +245,7 @@ A: The caller does not have access to that vector store. A: `file_citation` annotations require `file_id` metadata in search results. If your vector backend does not return file-level metadata, the answer text is still generated but citations can be empty. ---- + ## What to check next From bb8e0cd3e21aeea2cf23cbb3de28e37a1a492bfc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 22:47:45 +0530 Subject: [PATCH 19/83] Update 5.4 family values correctly --- ...odel_prices_and_context_window_backup.json | 32 +++++-------------- model_prices_and_context_window.json | 32 +++++-------------- 2 files changed, 16 insertions(+), 48 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4a627d4aa16..0be7aefa36c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18749,27 +18749,19 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_above_272k_tokens": 1.5e-07, - "cache_read_input_token_cost_flex": 3.9e-08, - "cache_read_input_token_cost_priority": 1.5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 3e-07, + "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_batches": 3.8e-08, "input_cost_per_token": 7.5e-07, - "input_cost_per_token_above_272k_tokens": 1.5e-06, "input_cost_per_token_flex": 3.75e-07, "input_cost_per_token_batches": 3.75e-07, - "input_cost_per_token_priority": 1.5e-06, - "input_cost_per_token_above_272k_tokens_priority": 3e-06, "litellm_provider": "openai", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, - "output_cost_per_token_above_272k_tokens": 6.75e-06, "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, - "output_cost_per_token_priority": 6.75e-06, - "output_cost_per_token_above_272k_tokens_priority": 1.0125e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18798,27 +18790,19 @@ }, "gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, - "cache_read_input_token_cost_above_272k_tokens": 4e-08, - "cache_read_input_token_cost_flex": 1.04e-08, - "cache_read_input_token_cost_priority": 4e-08, - "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_read_input_token_cost_flex": 1e-08, + "cache_read_input_token_cost_batches": 1e-08, "input_cost_per_token": 2e-07, - "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_batches": 1e-07, - "input_cost_per_token_priority": 4e-07, - "input_cost_per_token_above_272k_tokens_priority": 8e-07, "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, - "output_cost_per_token_above_272k_tokens": 1.875e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "output_cost_per_token_priority": 1.875e-06, - "output_cost_per_token_above_272k_tokens_priority": 2.8125e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4a627d4aa16..0be7aefa36c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18749,27 +18749,19 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_above_272k_tokens": 1.5e-07, - "cache_read_input_token_cost_flex": 3.9e-08, - "cache_read_input_token_cost_priority": 1.5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 3e-07, + "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_batches": 3.8e-08, "input_cost_per_token": 7.5e-07, - "input_cost_per_token_above_272k_tokens": 1.5e-06, "input_cost_per_token_flex": 3.75e-07, "input_cost_per_token_batches": 3.75e-07, - "input_cost_per_token_priority": 1.5e-06, - "input_cost_per_token_above_272k_tokens_priority": 3e-06, "litellm_provider": "openai", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, - "output_cost_per_token_above_272k_tokens": 6.75e-06, "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, - "output_cost_per_token_priority": 6.75e-06, - "output_cost_per_token_above_272k_tokens_priority": 1.0125e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18798,27 +18790,19 @@ }, "gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, - "cache_read_input_token_cost_above_272k_tokens": 4e-08, - "cache_read_input_token_cost_flex": 1.04e-08, - "cache_read_input_token_cost_priority": 4e-08, - "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_read_input_token_cost_flex": 1e-08, + "cache_read_input_token_cost_batches": 1e-08, "input_cost_per_token": 2e-07, - "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_batches": 1e-07, - "input_cost_per_token_priority": 4e-07, - "input_cost_per_token_above_272k_tokens_priority": 8e-07, "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, - "output_cost_per_token_above_272k_tokens": 1.875e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "output_cost_per_token_priority": 1.875e-06, - "output_cost_per_token_above_272k_tokens_priority": 2.8125e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", From 28506edd4951b2120779f0fefc745d36fc637dff Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 22:49:44 +0530 Subject: [PATCH 20/83] Add dodcs for gpt-4.5-mini --- .../blog/gpt_5_4_mini_nano/index.md | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 docs/my-website/blog/gpt_5_4_mini_nano/index.md diff --git a/docs/my-website/blog/gpt_5_4_mini_nano/index.md b/docs/my-website/blog/gpt_5_4_mini_nano/index.md new file mode 100644 index 00000000000..6d7c2b33f72 --- /dev/null +++ b/docs/my-website/blog/gpt_5_4_mini_nano/index.md @@ -0,0 +1,106 @@ +--- +slug: gpt_5_4_mini_nano +title: "Day 0 Support: GPT-5.4-mini and GPT-5.4-nano" +date: 2026-03-17T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +description: "GPT-5.4-mini and GPT-5.4-nano model support in LiteLLM" +tags: [openai, gpt-5.4-mini, gpt-5.4-nano, completion] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +LiteLLM now supports GPT-5.4-mini and GPT-5.4-nano — cost-effective models for simple completions and high-throughput workloads. + +:::note +If you're on **v1.82.3-stable** or above, you don't need any update to use these models. +::: + +## Usage + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: gpt-5.4-mini + litellm_params: + model: openai/gpt-5.4-mini + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-5.4-nano + litellm_params: + model: openai/gpt-5.4-nano + api_key: os.environ/OPENAI_API_KEY +``` + +**2. Start the proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it** + +```bash +# GPT-5.4-mini +curl -X POST "http://localhost:4000/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "gpt-5.4-mini", + "messages": [{"role": "user", "content": "What is the capital of France?"}] + }' + +# GPT-5.4-nano +curl -X POST "http://localhost:4000/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "gpt-5.4-nano", + "messages": [{"role": "user", "content": "What is 2 + 2?"}] + }' +``` + + + + +```python +from litellm import completion + +# GPT-5.4-mini +response = completion( + model="openai/gpt-5.4-mini", + messages=[{"role": "user", "content": "What is the capital of France?"}], +) +print(response.choices[0].message.content) + +# GPT-5.4-nano +response = completion( + model="openai/gpt-5.4-nano", + messages=[{"role": "user", "content": "What is 2 + 2?"}], +) +print(response.choices[0].message.content) +``` + + + + +## Notes + +- Both models support function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage. +- GPT-5.4-nano is the most cost-effective option for simple tasks; GPT-5.4-mini offers a balance of speed and capability. From 1b91e1656aba9f0ab99e62659ca520e5fd4eee41 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 23:20:59 +0530 Subject: [PATCH 21/83] Add support for gpt-5.4 mini and nano --- .../llms/openai/chat/gpt_5_transformation.py | 6 +- ...odel_prices_and_context_window_backup.json | 85 +++++++++++++------ model_prices_and_context_window.json | 85 +++++++++++++------ .../llms/openai/test_gpt5_transformation.py | 85 +++++++++++++++++++ 4 files changed, 202 insertions(+), 59 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index bb5783011a3..4fa45284df2 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -200,14 +200,14 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if "reasoning_effort" in optional_params: optional_params["reasoning_effort"] = normalized - if effective_effort is not None and effective_effort == "xhigh": - if not self._supports_reasoning_effort_level(model, "xhigh"): + if effective_effort is not None and effective_effort == "xhigh" or effective_effort == "minimal": + if not self._supports_reasoning_effort_level(model, effective_effort): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) else: raise litellm.utils.UnsupportedParamsError( message=( - "reasoning_effort='xhigh' is only supported for gpt-5.1-codex-max, gpt-5.2, and gpt-5.4+ models." + f"reasoning_effort={effective_effort} is not supported for this model." ), status_code=400, ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0be7aefa36c..1ac73d5bc0d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18333,7 +18333,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.2": { "cache_read_input_token_cost": 1.75e-07, @@ -18373,7 +18374,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.2-2025-12-11": { "cache_read_input_token_cost": 1.75e-07, @@ -18413,7 +18415,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.2-chat-latest": { "cache_read_input_token_cost": 1.75e-07, @@ -18450,7 +18453,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.3-chat-latest": { "cache_read_input_token_cost": 1.75e-07, @@ -18487,7 +18491,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.2-pro": { "input_cost_per_token": 2.1e-05, @@ -18520,7 +18525,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.2-pro-2025-12-11": { "input_cost_per_token": 2.1e-05, @@ -18553,7 +18559,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.4": { "cache_read_input_token_cost": 2.5e-07, @@ -18602,7 +18609,8 @@ "supports_service_tier": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, @@ -18697,7 +18705,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, @@ -18745,7 +18754,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, @@ -18786,7 +18796,8 @@ "supports_service_tier": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, @@ -18827,7 +18838,8 @@ "supports_service_tier": true, "supports_vision": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "gpt-5-pro": { "input_cost_per_token": 1.5e-05, @@ -18862,7 +18874,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-pro-2025-10-06": { "input_cost_per_token": 1.5e-05, @@ -18897,7 +18910,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, @@ -18939,7 +18953,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -18973,7 +18988,8 @@ "supports_tool_choice": false, "supports_vision": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-chat-latest": { "cache_read_input_token_cost": 1.25e-07, @@ -19007,7 +19023,8 @@ "supports_tool_choice": false, "supports_vision": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, @@ -19040,7 +19057,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex": { "cache_read_input_token_cost": 1.25e-07, @@ -19076,7 +19094,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex-max": { "cache_read_input_token_cost": 1.25e-07, @@ -19109,7 +19128,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex-mini": { "cache_read_input_token_cost": 2.5e-08, @@ -19145,7 +19165,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.2-codex": { "cache_read_input_token_cost": 1.75e-07, @@ -19181,7 +19202,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, @@ -19217,7 +19239,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, @@ -19259,7 +19282,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, @@ -19301,7 +19325,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-nano": { "cache_read_input_token_cost": 5e-09, @@ -19340,7 +19365,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, @@ -19378,7 +19404,9 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true + }, "gpt-image-1": { "cache_read_input_image_token_cost": 2.5e-06, @@ -36349,7 +36377,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-search-api-2025-10-14": { "cache_read_input_token_cost": 1.25e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0be7aefa36c..1ac73d5bc0d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18333,7 +18333,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.2": { "cache_read_input_token_cost": 1.75e-07, @@ -18373,7 +18374,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.2-2025-12-11": { "cache_read_input_token_cost": 1.75e-07, @@ -18413,7 +18415,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.2-chat-latest": { "cache_read_input_token_cost": 1.75e-07, @@ -18450,7 +18453,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.3-chat-latest": { "cache_read_input_token_cost": 1.75e-07, @@ -18487,7 +18491,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.2-pro": { "input_cost_per_token": 2.1e-05, @@ -18520,7 +18525,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.2-pro-2025-12-11": { "input_cost_per_token": 2.1e-05, @@ -18553,7 +18559,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.4": { "cache_read_input_token_cost": 2.5e-07, @@ -18602,7 +18609,8 @@ "supports_service_tier": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, @@ -18697,7 +18705,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, @@ -18745,7 +18754,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, @@ -18786,7 +18796,8 @@ "supports_service_tier": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, @@ -18827,7 +18838,8 @@ "supports_service_tier": true, "supports_vision": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "gpt-5-pro": { "input_cost_per_token": 1.5e-05, @@ -18862,7 +18874,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-pro-2025-10-06": { "input_cost_per_token": 1.5e-05, @@ -18897,7 +18910,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, @@ -18939,7 +18953,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -18973,7 +18988,8 @@ "supports_tool_choice": false, "supports_vision": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-chat-latest": { "cache_read_input_token_cost": 1.25e-07, @@ -19007,7 +19023,8 @@ "supports_tool_choice": false, "supports_vision": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, @@ -19040,7 +19057,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex": { "cache_read_input_token_cost": 1.25e-07, @@ -19076,7 +19094,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex-max": { "cache_read_input_token_cost": 1.25e-07, @@ -19109,7 +19128,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex-mini": { "cache_read_input_token_cost": 2.5e-08, @@ -19145,7 +19165,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.2-codex": { "cache_read_input_token_cost": 1.75e-07, @@ -19181,7 +19202,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, @@ -19217,7 +19239,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, @@ -19259,7 +19282,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, @@ -19301,7 +19325,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-nano": { "cache_read_input_token_cost": 5e-09, @@ -19340,7 +19365,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, @@ -19378,7 +19404,9 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true + }, "gpt-image-1": { "cache_read_input_image_token_cost": 2.5e-06, @@ -36349,7 +36377,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5-search-api-2025-10-14": { "cache_read_input_token_cost": 1.25e-07, diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 47ae3c44c9e..8ae9bf48a62 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -324,6 +324,91 @@ def test_gpt5_4_pro_allows_reasoning_effort_xhigh(config: OpenAIConfig): assert params["reasoning_effort"] == "xhigh" +def test_gpt5_4_allows_reasoning_effort_minimal(config: OpenAIConfig): + """gpt-5.4 supports reasoning_effort='minimal'.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model="gpt-5.4", + drop_params=False, + ) + assert params["reasoning_effort"] == "minimal" + + +def test_gpt5_4_pro_allows_reasoning_effort_minimal(config: OpenAIConfig): + """gpt-5.4-pro supports reasoning_effort='minimal'.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model="gpt-5.4-pro", + drop_params=False, + ) + assert params["reasoning_effort"] == "minimal" + + +def test_gpt5_4_mini_rejects_reasoning_effort_minimal(config: OpenAIConfig): + """gpt-5.4-mini does not support reasoning_effort='minimal'.""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model="gpt-5.4-mini", + drop_params=False, + ) + + +def test_gpt5_4_nano_rejects_reasoning_effort_minimal(config: OpenAIConfig): + """gpt-5.4-nano does not support reasoning_effort='minimal'.""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model="gpt-5.4-nano", + drop_params=False, + ) + + +def test_gpt5_drops_reasoning_effort_minimal_when_requested(config: OpenAIConfig): + """reasoning_effort='minimal' is dropped for unsupported models when drop_params=True.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model="gpt-5.4-mini", + drop_params=True, + ) + assert "reasoning_effort" not in params + + +def test_gpt5_minimal_dict_triggers_validation(config: OpenAIConfig): + """Dict with effort='minimal' triggers minimal model-support validation.""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "minimal", "summary": "detailed"}}, + optional_params={}, + model="gpt-5.4-mini", + drop_params=False, + ) + + +def test_gpt5_minimal_dict_accepted_for_supported_model(config: OpenAIConfig): + """Dict with effort='minimal' passes through for gpt-5.4+.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "minimal", "summary": "detailed"}}, + optional_params={}, + model="gpt-5.4", + drop_params=False, + ) + assert params["reasoning_effort"] == "minimal" + + +def test_gpt5_supports_reasoning_effort_level_minimal(gpt5_config: OpenAIGPT5Config): + """Test that _supports_reasoning_effort_level correctly identifies minimal support.""" + assert gpt5_config._supports_reasoning_effort_level("gpt-5.4", "minimal") + assert gpt5_config._supports_reasoning_effort_level("gpt-5.4-pro", "minimal") + assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-mini", "minimal") + assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-nano", "minimal") + + def test_gpt5_normalizes_reasoning_effort_dict_with_summary(config: OpenAIConfig): """Dict with summary/generate_summary is normalized for chat completions.""" params = config.map_openai_params( From 0ecea85bd0e6f4187f22176f7317d3fb1f498847 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 23:27:01 +0530 Subject: [PATCH 22/83] Fix supports none flag --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1ac73d5bc0d..c80d6372332 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18837,7 +18837,7 @@ "supports_tool_choice": true, "supports_service_tier": true, "supports_vision": true, - "supports_none_reasoning_effort": false, + "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1ac73d5bc0d..c80d6372332 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18837,7 +18837,7 @@ "supports_tool_choice": true, "supports_service_tier": true, "supports_vision": true, - "supports_none_reasoning_effort": false, + "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, From a34f7e483112c7a4b958cf19debc50e891fdd52c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Mar 2026 23:27:39 +0530 Subject: [PATCH 23/83] Fix paranthesis: --- litellm/llms/openai/chat/gpt_5_transformation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 4fa45284df2..8522a97a387 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -200,7 +200,9 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if "reasoning_effort" in optional_params: optional_params["reasoning_effort"] = normalized - if effective_effort is not None and effective_effort == "xhigh" or effective_effort == "minimal": + if effective_effort is not None and ( + effective_effort == "xhigh" or effective_effort == "minimal" + ): if not self._supports_reasoning_effort_level(model, effective_effort): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) From 0564e9547b130587d286ae57d3cb6fe83c07eea8 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 08:37:31 +0530 Subject: [PATCH 24/83] Fix greptile comments --- litellm/model_prices_and_context_window_backup.json | 13 ++++++++----- model_prices_and_context_window.json | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c80d6372332..2cc63523daa 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3435,7 +3435,8 @@ "supports_tool_choice": true, "supports_service_tier": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "azure/gpt-5.1-chat-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -18217,7 +18218,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.1": { "cache_read_input_token_cost": 1.25e-07, @@ -18256,7 +18258,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -18295,7 +18298,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.1-chat-latest": { "cache_read_input_token_cost": 1.25e-07, @@ -19406,7 +19410,6 @@ "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true - }, "gpt-image-1": { "cache_read_input_image_token_cost": 2.5e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c80d6372332..2cc63523daa 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3435,7 +3435,8 @@ "supports_tool_choice": true, "supports_service_tier": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "azure/gpt-5.1-chat-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -18217,7 +18218,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.1": { "cache_read_input_token_cost": 1.25e-07, @@ -18256,7 +18258,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -18295,7 +18298,8 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, - "supports_xhigh_reasoning_effort": false + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "gpt-5.1-chat-latest": { "cache_read_input_token_cost": 1.25e-07, @@ -19406,7 +19410,6 @@ "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true - }, "gpt-image-1": { "cache_read_input_image_token_cost": 2.5e-06, From c20c465a028db110d346e6f0a123677bb74262ad Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 08:55:40 +0530 Subject: [PATCH 25/83] greptile comments --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2cc63523daa..44fe6a06ad1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18763,7 +18763,7 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_batches": 3.8e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_flex": 3.75e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2cc63523daa..44fe6a06ad1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18763,7 +18763,7 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_batches": 3.8e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_flex": 3.75e-07, From 6fe3188af048905504277d9566336a0a312d95a1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 09:04:00 +0530 Subject: [PATCH 26/83] fix(azure-ai-agents): accumulate annotations from multiple text items in streaming - Fix bug where only last text item's annotations were preserved when thread.message.completed contained multiple text content items - Accumulate annotations via extend() instead of overwriting - Add test_azure_ai_agents_streaming_annotations_from_completed_message - Add test_azure_ai_agents_streaming_accumulates_annotations_from_multiple_text_items Addresses Greptile review on PR #23849 Made-with: Cursor --- litellm/llms/azure_ai/agents/handler.py | 4 +- tests/llm_translation/test_azure_agents.py | 162 +++++++++++++++++++++ 2 files changed, 165 insertions(+), 1 deletion(-) diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index 5b779acb0d1..95c0a4c577c 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -700,7 +700,9 @@ class AzureAIAgentsHandler: raw_annotations ) if transformed: - collected_annotations = transformed + if collected_annotations is None: + collected_annotations = [] + collected_annotations.extend(transformed) # Process message deltas - this is where the actual content comes if current_event == "thread.message.delta": diff --git a/tests/llm_translation/test_azure_agents.py b/tests/llm_translation/test_azure_agents.py index 19ce49a3bc9..3e6b1e00a79 100644 --- a/tests/llm_translation/test_azure_agents.py +++ b/tests/llm_translation/test_azure_agents.py @@ -23,12 +23,14 @@ Example environment variables: See: https://learn.microsoft.com/en-us/azure/ai-foundry/agents/quickstart """ +import json import os import sys sys.path.insert(0, os.path.abspath("../..")) import pytest +from unittest.mock import MagicMock import litellm @@ -465,6 +467,166 @@ def test_azure_ai_agents_build_model_response_without_annotations(): assert getattr(result.choices[0].message, "annotations", None) is None +@pytest.mark.asyncio +async def test_azure_ai_agents_streaming_annotations_from_completed_message(): + """ + Test that annotations from thread.message.completed SSE events are collected + and attached to the final chunk's delta. + + Ref: https://github.com/BerriAI/litellm/issues/19126 + """ + from litellm.llms.azure_ai.agents.handler import AzureAIAgentsHandler + + handler = AzureAIAgentsHandler() + + # SSE lines simulating a stream with annotations in thread.message.completed + completed_data = { + "content": [ + { + "type": "text", + "text": { + "value": "According to [1], the answer is 42.", + "annotations": [ + { + "type": "url_citation", + "text": "[1]", + "start_index": 12, + "end_index": 15, + "url_citation": { + "url": "https://example.com/citation", + "title": "Citation Source", + }, + } + ], + }, + } + ] + } + + sse_lines = [ + "event: thread.created", + "", + 'data: {"id": "thread_stream_123"}', + "", + "event: thread.message.delta", + "", + 'data: {"delta": {"content": [{"type": "text", "text": {"value": "According to [1], the answer is 42."}}]}}', + "", + "event: thread.message.completed", + "", + f"data: {json.dumps(completed_data)}", + "", + "data: [DONE]", + ] + + async def mock_aiter_lines(): + for line in sse_lines: + yield line + + mock_response = MagicMock() + mock_response.aiter_lines = MagicMock(return_value=mock_aiter_lines()) + + chunks = [] + async for chunk in handler._process_sse_stream(mock_response, "azure_ai/agents/asst_123"): + chunks.append(chunk) + + # Should have content chunks + final [DONE] chunk + assert len(chunks) >= 1 + final_chunk = chunks[-1] + assert final_chunk.choices[0].finish_reason == "stop" + assert final_chunk.choices[0].delta.annotations is not None + assert len(final_chunk.choices[0].delta.annotations) == 1 + ann = final_chunk.choices[0].delta.annotations[0] + assert ann["type"] == "url_citation" + assert ann["url_citation"]["url"] == "https://example.com/citation" + assert ann["url_citation"]["title"] == "Citation Source" + + +@pytest.mark.asyncio +async def test_azure_ai_agents_streaming_accumulates_annotations_from_multiple_text_items(): + """ + Test that annotations from multiple text content items in thread.message.completed + are accumulated (not overwritten). + + Ref: Greptile review on PR #23849 + """ + from litellm.llms.azure_ai.agents.handler import AzureAIAgentsHandler + + handler = AzureAIAgentsHandler() + + # Two text blocks, each with distinct citations + completed_data = { + "content": [ + { + "type": "text", + "text": { + "value": "First source [1].", + "annotations": [ + { + "type": "url_citation", + "text": "[1]", + "start_index": 12, + "end_index": 15, + "url_citation": { + "url": "https://example.com/first", + "title": "First", + }, + } + ], + }, + }, + { + "type": "text", + "text": { + "value": "Second source [2].", + "annotations": [ + { + "type": "url_citation", + "text": "[2]", + "start_index": 13, + "end_index": 16, + "url_citation": { + "url": "https://example.com/second", + "title": "Second", + }, + } + ], + }, + }, + ] + } + + sse_lines = [ + "event: thread.created", + "", + 'data: {"id": "thread_multi"}', + "", + "event: thread.message.completed", + "", + f"data: {json.dumps(completed_data)}", + "", + "data: [DONE]", + ] + + async def mock_aiter_lines(): + for line in sse_lines: + yield line + + mock_response = MagicMock() + mock_response.aiter_lines = MagicMock(return_value=mock_aiter_lines()) + + chunks = [] + async for chunk in handler._process_sse_stream(mock_response, "azure_ai/agents/asst_123"): + chunks.append(chunk) + + final_chunk = chunks[-1] + assert final_chunk.choices[0].delta.annotations is not None + assert len(final_chunk.choices[0].delta.annotations) == 2 + urls = [a["url_citation"]["url"] for a in final_chunk.choices[0].delta.annotations] + assert "https://example.com/first" in urls + assert "https://example.com/second" in urls + + @pytest.mark.asyncio async def test_azure_ai_agents_conversation_continuity(): """ From 6514446dcb1984c29cf9fd61bcd3627b3cff0579 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 09:09:30 +0530 Subject: [PATCH 27/83] Update litellm/llms/azure_ai/agents/handler.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/llms/azure_ai/agents/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index 95c0a4c577c..3fbda13d3c7 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -137,7 +137,7 @@ class AzureAIAgentsHandler: result: List[Dict[str, Any]] = [] for ann in raw_annotations: - ann_type = ann.get("type", "url_citation") + ann_type = ann.get("type") if ann_type == "url_citation": url_citation = dict(ann.get("url_citation", {})) # Azure puts start/end_index at annotation level; OpenAI From 018ccff23f1df9e4d223741e0f084447f08bcade Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 09:11:20 +0530 Subject: [PATCH 28/83] fix(vertex-ai): address greptile review feedback on batch cancel - Add try/except httpx.HTTPStatusError blocks in _async_cancel_batch for both POST cancel and GET retrieve calls, with verbose_logger error logging - Fix endpoint extraction inconsistency: compute endpoint from URL without :cancel suffix so it matches behaviour of create_batch/retrieve_batch - Add explicit validation that api_base ends with ':cancel' before stripping it, raising a descriptive error for unsupported custom proxy URL rewriting scenarios - Use string-based patch() in test instead of patch.object() for robustness against import order changes Made-with: Cursor --- litellm/llms/vertex_ai/batches/handler.py | 54 ++++++++++++++----- .../test_vertex_ai_batch_transformation.py | 2 +- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index c7b9287c080..36728499b9c 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -399,10 +399,12 @@ class VertexAIBatchPrediction(VertexLLM): vertex_project=vertex_project or project_id, ) - default_api_base = f"{default_api_base}/{batch_id}:cancel" + # Compute endpoint from the URL without :cancel for consistency with other methods + base_without_cancel = f"{default_api_base}/{batch_id}" + default_api_base = f"{base_without_cancel}:cancel" - if len(default_api_base.split(":")) > 1: - endpoint = default_api_base.split(":")[-1] + if len(base_without_cancel.split(":")) > 1: + endpoint = base_without_cancel.split(":")[-1] else: endpoint = "" @@ -420,6 +422,14 @@ class VertexAIBatchPrediction(VertexLLM): vertex_api_version="v1", ) + if not api_base.endswith(":cancel"): + raise ValueError( + f"cancel_batch: expected api_base to end with ':cancel', got: {api_base!r}. " + "Custom proxy URL rewriting is not supported for this operation." + ) + + retrieve_api_base = api_base.rsplit(":cancel", 1)[0] + headers = { "Content-Type": "application/json; charset=utf-8", "Authorization": f"Bearer {access_token}", @@ -428,7 +438,7 @@ class VertexAIBatchPrediction(VertexLLM): if _is_async is True: return self._async_cancel_batch( api_base=api_base, - retrieve_api_base=api_base.rsplit(":cancel", 1)[0], + retrieve_api_base=retrieve_api_base, headers=headers, ) @@ -443,7 +453,7 @@ class VertexAIBatchPrediction(VertexLLM): raise Exception(f"Error: {response.status_code} {response.text}") retrieve_response = sync_handler.get( - url=api_base.rsplit(":cancel", 1)[0], + url=retrieve_api_base, headers=headers, ) if retrieve_response.status_code != 200: @@ -466,18 +476,34 @@ class VertexAIBatchPrediction(VertexLLM): client = get_async_httpx_client( llm_provider=litellm.LlmProviders.VERTEX_AI, ) - response = await client.post( - url=api_base, - headers=headers, - data=json.dumps({}), - ) + try: + response = await client.post( + url=api_base, + headers=headers, + data=json.dumps({}), + ) + except httpx.HTTPStatusError as e: + litellm.verbose_logger.error( + "Vertex AI batch cancel failed: status=%s, body=%s", + e.response.status_code, + e.response.text[:1000], + ) + raise if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - retrieve_response = await client.get( - url=retrieve_api_base, - headers=headers, - ) + try: + retrieve_response = await client.get( + url=retrieve_api_base, + headers=headers, + ) + except httpx.HTTPStatusError as e: + litellm.verbose_logger.error( + "Vertex AI batch retrieve-after-cancel failed: status=%s, body=%s", + e.response.status_code, + e.response.text[:1000], + ) + raise if retrieve_response.status_code != 200: raise Exception( f"Error: {retrieve_response.status_code} {retrieve_response.text}" diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py index d27cd8ba8a7..e03555a9c09 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py @@ -101,7 +101,7 @@ async def test_litellm_cancel_batch_vertex_ai(): mock_response.id = "batch_123" mock_response.status = "cancelling" - with patch.object(litellm.batches.main, "vertex_ai_batches_instance") as mock_instance: + with patch("litellm.batches.main.vertex_ai_batches_instance") as mock_instance: mock_instance.cancel_batch.return_value = mock_response response = litellm.cancel_batch( From 52bf372319b959c49be126472268cc0b7bf1bb8a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 09:17:57 +0530 Subject: [PATCH 29/83] fix(gpt5): treat missing supports_minimal_reasoning_effort as supported Add _is_reasoning_effort_level_explicitly_disabled to use opt-out semantics for minimal effort: unknown/unlisted models pass through, only blocked when the model map explicitly sets supports_minimal_reasoning_effort=false. xhigh keeps opt-in semantics (must be explicitly supported). Adds test for unknown-model passthrough and explicit-disabled detection. Made-with: Cursor --- .../llms/openai/chat/gpt_5_transformation.py | 42 +++++++++++++++++-- .../llms/openai/test_gpt5_transformation.py | 35 ++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 8522a97a387..6291f4232d5 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -3,7 +3,7 @@ from typing import Optional, Union import litellm -from litellm.utils import _supports_factory +from litellm.utils import _get_model_cost_key, _get_model_info_helper, _supports_factory from .gpt_transformation import OpenAIGPTConfig @@ -113,6 +113,28 @@ class OpenAIGPT5Config(OpenAIGPTConfig): key=f"supports_{level}_reasoning_effort", ) + @classmethod + def _is_reasoning_effort_level_explicitly_disabled( + cls, model: str, level: str + ) -> bool: + """Return True only when the model map explicitly sets the capability to False. + + Unlike ``_supports_reasoning_effort_level`` (which requires an explicit True), + this method returns True only when ``supports_{level}_reasoning_effort`` is + explicitly set to ``False`` in the model map. A missing key is treated as + supported (i.e. this method returns False = not disabled). + + Use this for opt-out checks where unknown models should be allowed through. + """ + try: + key = f"supports_{level}_reasoning_effort" + cost_key = _get_model_cost_key(model) + entry = litellm.model_cost.get(cost_key or model) or {} + val = entry.get(key) + return val is False + except Exception: + return False + def get_supported_openai_params(self, model: str) -> list: if self.is_model_gpt_5_search_model(model): return [ @@ -200,9 +222,8 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if "reasoning_effort" in optional_params: optional_params["reasoning_effort"] = normalized - if effective_effort is not None and ( - effective_effort == "xhigh" or effective_effort == "minimal" - ): + if effective_effort == "xhigh": + # xhigh is an opt-in capability: only allow if model explicitly supports it. if not self._supports_reasoning_effort_level(model, effective_effort): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) @@ -213,6 +234,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig): ), status_code=400, ) + elif effective_effort == "minimal": + # minimal is opt-out: unknown models pass through; only block when + # the model map explicitly sets supports_minimal_reasoning_effort=false. + if self._is_reasoning_effort_level_explicitly_disabled(model, effective_effort): + if litellm.drop_params or drop_params: + non_default_params.pop("reasoning_effort", None) + else: + raise litellm.utils.UnsupportedParamsError( + message=( + f"reasoning_effort={effective_effort} is not supported for this model." + ), + status_code=400, + ) ################################################################ # max_tokens is not supported for gpt-5 models on OpenAI API diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 8ae9bf48a62..535fba16140 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -409,6 +409,41 @@ def test_gpt5_supports_reasoning_effort_level_minimal(gpt5_config: OpenAIGPT5Con assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-nano", "minimal") +def test_gpt5_minimal_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config): + """_is_reasoning_effort_level_explicitly_disabled returns True only for explicit False entries. + + Models with supports_minimal_reasoning_effort=false → disabled. + Models with supports_minimal_reasoning_effort=true (or missing) → not disabled. + """ + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.4-mini", "minimal" + ) + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.4-nano", "minimal" + ) + assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.4", "minimal" + ) + assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.4-pro", "minimal" + ) + + +def test_gpt5_unknown_model_passes_through_minimal(config: OpenAIConfig): + """Unknown/unlisted gpt-5 models should pass reasoning_effort='minimal' through. + + Missing supports_minimal_reasoning_effort key is treated as supported, + not as unsupported, to avoid breaking custom or newly-announced models. + """ + params = config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model="gpt-5.4-turbo-preview", + drop_params=False, + ) + assert params["reasoning_effort"] == "minimal" + + def test_gpt5_normalizes_reasoning_effort_dict_with_summary(config: OpenAIConfig): """Dict with summary/generate_summary is normalized for chat completions.""" params = config.map_openai_params( From a41239cb966341f78c6a3da96dd53c3332929c8f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 09:24:34 +0530 Subject: [PATCH 30/83] greptile comments --- litellm/model_prices_and_context_window_backup.json | 2 ++ model_prices_and_context_window.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 44fe6a06ad1..b5330e58556 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18799,6 +18799,7 @@ "supports_tool_choice": true, "supports_service_tier": true, "supports_vision": true, + "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false @@ -18841,6 +18842,7 @@ "supports_tool_choice": true, "supports_service_tier": true, "supports_vision": true, + "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 44fe6a06ad1..b5330e58556 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18799,6 +18799,7 @@ "supports_tool_choice": true, "supports_service_tier": true, "supports_vision": true, + "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false @@ -18841,6 +18842,7 @@ "supports_tool_choice": true, "supports_service_tier": true, "supports_vision": true, + "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false From 74382f1c89cf91b3ddc58dd82d6298407465c51c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 09:28:19 +0530 Subject: [PATCH 31/83] fix(vertex-ai): address greptile review feedback on batch cancel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace misleading endpoint extraction with explicit endpoint = "cancel" - Compute retrieve_api_base from URL components directly instead of stripping ":cancel" from the post-proxy URL, removing the hard ValueError that broke any custom Vertex AI proxy configuration - Align cancel_batch provider priority in proxy endpoints to match create_batch order: body field → request headers → query params → default Made-with: Cursor --- litellm/llms/vertex_ai/batches/handler.py | 21 +++++++------------- litellm/proxy/batches_endpoints/endpoints.py | 2 +- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index 36728499b9c..b30130688d2 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -399,14 +399,11 @@ class VertexAIBatchPrediction(VertexLLM): vertex_project=vertex_project or project_id, ) - # Compute endpoint from the URL without :cancel for consistency with other methods - base_without_cancel = f"{default_api_base}/{batch_id}" - default_api_base = f"{base_without_cancel}:cancel" + retrieve_api_base_default = f"{default_api_base}/{batch_id}" + default_api_base = f"{retrieve_api_base_default}:cancel" - if len(base_without_cancel.split(":")) > 1: - endpoint = base_without_cancel.split(":")[-1] - else: - endpoint = "" + # The Vertex AI action suffix for this operation + endpoint = "cancel" _, api_base = self._check_custom_proxy( api_base=api_base, @@ -422,13 +419,9 @@ class VertexAIBatchPrediction(VertexLLM): vertex_api_version="v1", ) - if not api_base.endswith(":cancel"): - raise ValueError( - f"cancel_batch: expected api_base to end with ':cancel', got: {api_base!r}. " - "Custom proxy URL rewriting is not supported for this operation." - ) - - retrieve_api_base = api_base.rsplit(":cancel", 1)[0] + # Use the canonical retrieve URL built from components rather than stripping + # ":cancel" from api_base, so custom proxy URL rewriting does not break retrieval. + retrieve_api_base = retrieve_api_base_default headers = { "Content-Type": "application/json; charset=utf-8", diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 9ce1b6e9168..3b26e6c096c 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -898,9 +898,9 @@ async def cancel_batch( else: custom_llm_provider = ( provider + or data.pop("custom_llm_provider", None) or get_custom_llm_provider_from_request_headers(request=request) or get_custom_llm_provider_from_request_query(request=request) - or data.pop("custom_llm_provider", None) or "openai" ) # Extract batch_id from data to avoid "multiple values for keyword argument" error From 5dd89f16f5369c65af33600f43ee22a4e39b9de6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 09:37:19 +0530 Subject: [PATCH 32/83] address greptile review: remove unused import, normalize model lookup, add xhigh tests - Remove unused _get_model_info_helper import - Normalize model via get_llm_provider in _is_reasoning_effort_level_explicitly_disabled so provider-prefixed names (openai/gpt-5.4-mini) resolve correctly - Add test_gpt5_4_mini_allows_reasoning_effort_xhigh - Add test_gpt5_4_nano_allows_reasoning_effort_xhigh - Add test_gpt5_4_mini_provider_prefixed_rejects_minimal - Extend test_gpt5_minimal_explicitly_disabled_check for openai/gpt-5.4-mini --- .../llms/openai/chat/gpt_5_transformation.py | 11 ++++-- .../llms/openai/test_gpt5_transformation.py | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 6291f4232d5..60a21e19a9f 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -3,7 +3,7 @@ from typing import Optional, Union import litellm -from litellm.utils import _get_model_cost_key, _get_model_info_helper, _supports_factory +from litellm.utils import _get_model_cost_key, _supports_factory from .gpt_transformation import OpenAIGPTConfig @@ -125,11 +125,16 @@ class OpenAIGPT5Config(OpenAIGPTConfig): supported (i.e. this method returns False = not disabled). Use this for opt-out checks where unknown models should be allowed through. + Normalizes the model via get_llm_provider so provider-prefixed names + (e.g. openai/gpt-5.4-mini) resolve correctly. """ try: + normalized_model, _, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=None + ) key = f"supports_{level}_reasoning_effort" - cost_key = _get_model_cost_key(model) - entry = litellm.model_cost.get(cost_key or model) or {} + cost_key = _get_model_cost_key(normalized_model) + entry = litellm.model_cost.get(cost_key or normalized_model) or {} val = entry.get(key) return val is False except Exception: diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 535fba16140..72e8a0c185d 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -324,6 +324,28 @@ def test_gpt5_4_pro_allows_reasoning_effort_xhigh(config: OpenAIConfig): assert params["reasoning_effort"] == "xhigh" +def test_gpt5_4_mini_allows_reasoning_effort_xhigh(config: OpenAIConfig): + """gpt-5.4-mini supports reasoning_effort='xhigh'.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model="gpt-5.4-mini", + drop_params=False, + ) + assert params["reasoning_effort"] == "xhigh" + + +def test_gpt5_4_nano_allows_reasoning_effort_xhigh(config: OpenAIConfig): + """gpt-5.4-nano supports reasoning_effort='xhigh'.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model="gpt-5.4-nano", + drop_params=False, + ) + assert params["reasoning_effort"] == "xhigh" + + def test_gpt5_4_allows_reasoning_effort_minimal(config: OpenAIConfig): """gpt-5.4 supports reasoning_effort='minimal'.""" params = config.map_openai_params( @@ -368,6 +390,17 @@ def test_gpt5_4_nano_rejects_reasoning_effort_minimal(config: OpenAIConfig): ) +def test_gpt5_4_mini_provider_prefixed_rejects_minimal(config: OpenAIConfig): + """openai/gpt-5.4-mini correctly rejects minimal (model lookup normalizes prefix).""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model="openai/gpt-5.4-mini", + drop_params=False, + ) + + def test_gpt5_drops_reasoning_effort_minimal_when_requested(config: OpenAIConfig): """reasoning_effort='minimal' is dropped for unsupported models when drop_params=True.""" params = config.map_openai_params( @@ -414,6 +447,7 @@ def test_gpt5_minimal_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config): Models with supports_minimal_reasoning_effort=false → disabled. Models with supports_minimal_reasoning_effort=true (or missing) → not disabled. + Provider-prefixed models (openai/gpt-5.4-mini) are normalized before lookup. """ assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( "gpt-5.4-mini", "minimal" @@ -421,6 +455,9 @@ def test_gpt5_minimal_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config): assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( "gpt-5.4-nano", "minimal" ) + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "openai/gpt-5.4-mini", "minimal" + ) assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled( "gpt-5.4", "minimal" ) From 74ae17d15305334d0614b107e866bb5f6f7ef0e9 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 09:41:46 +0530 Subject: [PATCH 33/83] greptile comments --- litellm/batches/main.py | 2 +- litellm/llms/vertex_ai/batches/handler.py | 31 +++++++++++++------ .../test_vertex_ai_batch_transformation.py | 2 +- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 36093d071ba..ae79469dd10 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -1048,7 +1048,7 @@ def cancel_batch( litellm_params=litellm_params, ) elif custom_llm_provider == "vertex_ai": - api_base = optional_params.api_base or "" + api_base = optional_params.api_base or None vertex_ai_project = ( optional_params.vertex_project or litellm.vertex_project diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index b30130688d2..f4ba0533c81 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -400,28 +400,41 @@ class VertexAIBatchPrediction(VertexLLM): ) retrieve_api_base_default = f"{default_api_base}/{batch_id}" - default_api_base = f"{retrieve_api_base_default}:cancel" + cancel_api_base_default = f"{retrieve_api_base_default}:cancel" - # The Vertex AI action suffix for this operation - endpoint = "cancel" + # Save the caller-supplied value before _check_custom_proxy overwrites api_base, + # so we can pass it unchanged to the second proxy-check for the retrieve URL. + caller_api_base = api_base _, api_base = self._check_custom_proxy( - api_base=api_base, + api_base=caller_api_base, custom_llm_provider="vertex_ai", gemini_api_key=None, - endpoint=endpoint, + endpoint="cancel", stream=None, auth_header=None, - url=default_api_base, + url=cancel_api_base_default, model=None, vertex_project=vertex_project or project_id, vertex_location=vertex_location or "us-central1", vertex_api_version="v1", ) - # Use the canonical retrieve URL built from components rather than stripping - # ":cancel" from api_base, so custom proxy URL rewriting does not break retrieval. - retrieve_api_base = retrieve_api_base_default + # Route the retrieve GET through the same proxy as the cancel POST by running + # _check_custom_proxy a second time with the non-cancel default URL. + _, retrieve_api_base = self._check_custom_proxy( + api_base=caller_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="", + stream=None, + auth_header=None, + url=retrieve_api_base_default, + model=None, + vertex_project=vertex_project or project_id, + vertex_location=vertex_location or "us-central1", + vertex_api_version="v1", + ) headers = { "Content-Type": "application/json; charset=utf-8", diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py index e03555a9c09..1cf6fa32661 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py @@ -45,7 +45,7 @@ def test_output_file_id_falls_back_to_output_uri_prefix_with_predictions_jsonl() @pytest.mark.asyncio -async def test_vertex_ai_cancel_batch(): +def test_vertex_ai_cancel_batch(): """Test that vertex_ai cancel_batch calls the correct API endpoint""" handler = VertexAIBatchPrediction(gcs_bucket_name="test-bucket") From d0d593beb8027b2b467b1ef9cbdd302981d658a4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 09:48:24 +0530 Subject: [PATCH 34/83] Update tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../llms/vertex_ai/test_vertex_ai_batch_transformation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py index 1cf6fa32661..1fa8dd2e16c 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py @@ -44,7 +44,6 @@ def test_output_file_id_falls_back_to_output_uri_prefix_with_predictions_jsonl() ) -@pytest.mark.asyncio def test_vertex_ai_cancel_batch(): """Test that vertex_ai cancel_batch calls the correct API endpoint""" handler = VertexAIBatchPrediction(gcs_bucket_name="test-bucket") From e46dd949f2b10ccbeabf4d75ca89fec1574e7d23 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 09:58:20 +0530 Subject: [PATCH 35/83] Add test for reasoning effort none --- .../llms/openai/test_gpt5_transformation.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 72e8a0c185d..a173d2015bc 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -345,6 +345,25 @@ def test_gpt5_4_nano_allows_reasoning_effort_xhigh(config: OpenAIConfig): ) assert params["reasoning_effort"] == "xhigh" +def test_gpt5_4_nano_allows_reasoning_effort_none(config: OpenAIConfig): + """gpt-5.4-nano supports reasoning_effort='none'.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": "none"}, + optional_params={}, + model="gpt-5.4-nano", + drop_params=False, + ) + assert params["reasoning_effort"] == "none" + +def test_gpt5_4_mini_allows_reasoning_effort_none(config: OpenAIConfig): + """gpt-5.4-mini supports reasoning_effort='none'.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": "none"}, + optional_params={}, + model="gpt-5.4-mini", + drop_params=False, + ) + assert params["reasoning_effort"] == "none" def test_gpt5_4_allows_reasoning_effort_minimal(config: OpenAIConfig): """gpt-5.4 supports reasoning_effort='minimal'.""" From 547db8f5d1f1408e598088380fa007497efb3f23 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 10:01:47 +0530 Subject: [PATCH 36/83] Fix greptile comments --- litellm/llms/vertex_ai/batches/handler.py | 24 +++++------------------ 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index f4ba0533c81..9c760f628ed 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -402,12 +402,8 @@ class VertexAIBatchPrediction(VertexLLM): retrieve_api_base_default = f"{default_api_base}/{batch_id}" cancel_api_base_default = f"{retrieve_api_base_default}:cancel" - # Save the caller-supplied value before _check_custom_proxy overwrites api_base, - # so we can pass it unchanged to the second proxy-check for the retrieve URL. - caller_api_base = api_base - _, api_base = self._check_custom_proxy( - api_base=caller_api_base, + api_base=api_base, custom_llm_provider="vertex_ai", gemini_api_key=None, endpoint="cancel", @@ -420,20 +416,10 @@ class VertexAIBatchPrediction(VertexLLM): vertex_api_version="v1", ) - # Route the retrieve GET through the same proxy as the cancel POST by running - # _check_custom_proxy a second time with the non-cancel default URL. - _, retrieve_api_base = self._check_custom_proxy( - api_base=caller_api_base, - custom_llm_provider="vertex_ai", - gemini_api_key=None, - endpoint="", - stream=None, - auth_header=None, - url=retrieve_api_base_default, - model=None, - vertex_project=vertex_project or project_id, - vertex_location=vertex_location or "us-central1", - vertex_api_version="v1", + retrieve_api_base = ( + api_base.removesuffix(":cancel") + if api_base.endswith(":cancel") + else retrieve_api_base_default ) headers = { From dc7b7f852d11f113b61df87be757d094fa306c24 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 10:10:29 +0530 Subject: [PATCH 37/83] =?UTF-8?q?fix(file=5Fsearch):=20address=20greptile?= =?UTF-8?q?=20review=20=E2=80=94=20dead=20code,=20follow-up=20context,=20c?= =?UTF-8?q?ost=20tracking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove dead `should_use_emulated_file_search` (main.py uses its own inline guard) - Remove dead `fallback_vector_store_ids` param from `_run_vector_searches` - Include all first_response.output items in follow_up_input so text blocks/reasoning from providers like Anthropic aren't dropped from conversation context - Accumulate first provider call's response_cost into synthesized _hidden_params so billing callbacks see the total cost of both emulated-flow LLM calls - Remove broad tools=[] filter from transformation.py (backward-incompatible); the follow-up call already passes tools=None which is filtered by the v is not None guard Made-with: Cursor --- .../responses/file_search/emulated_handler.py | 64 ++++++------------- .../transformation.py | 6 +- 2 files changed, 23 insertions(+), 47 deletions(-) diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 5d2c23fdfa3..16860218841 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -26,25 +26,6 @@ ToolParam = Any FILE_SEARCH_FUNCTION_NAME = "litellm_file_search" -# --------------------------------------------------------------------------- -# Detection -# --------------------------------------------------------------------------- - -def should_use_emulated_file_search( - tools: Optional[Iterable[ToolParam]], - provider_config: Any, # BaseResponsesAPIConfig -) -> bool: - """Return True when there is a file_search tool and the provider can't handle it natively.""" - if not tools: - return False - has_fs = any( - isinstance(t, dict) and t.get("type") == "file_search" for t in tools - ) - if not has_fs: - return False - return provider_config is None or not provider_config.supports_native_file_search() - - # --------------------------------------------------------------------------- # Tool conversion # --------------------------------------------------------------------------- @@ -126,15 +107,13 @@ def _replace_file_search_tools( async def _run_vector_searches( queries: List[str], vector_store_ids: List[str], - fallback_vector_store_ids: List[str], ) -> Tuple[List[str], List[VectorStoreSearchResult]]: """ Run `asearch` against all vector stores for all queries and collect results. Args: queries: List of search queries to execute (like OpenAI's multi-query approach) - vector_store_ids: Specific vector store IDs to search - fallback_vector_store_ids: Fallback IDs if vector_store_ids is empty + vector_store_ids: Vector store IDs to search Returns: (queries_list, combined_results) @@ -142,7 +121,7 @@ async def _run_vector_searches( import litellm.vector_stores.main as vs_main all_results: List[VectorStoreSearchResult] = [] - ids_to_search = vector_store_ids or fallback_vector_store_ids + ids_to_search = vector_store_ids # Execute each query against all vector stores for query in queries: @@ -337,11 +316,16 @@ def _synthesize_responses_api_response( original_response: ResponsesAPIResponse, file_search_call_output: Dict[str, Any], message_output: Dict[str, Any], + first_response: Optional[ResponsesAPIResponse] = None, ) -> ResponsesAPIResponse: """ Return a new ResponsesAPIResponse with: output[0] = file_search_call item output[1] = message item (with citations) + + When first_response is provided, its response_cost is accumulated into the + synthesized _hidden_params so that billing callbacks see the total cost of + both provider calls that the emulated flow makes. """ synthesized = ResponsesAPIResponse( id=getattr(original_response, "id", f"resp_{uuid.uuid4().hex}"), @@ -354,7 +338,14 @@ def _synthesize_responses_api_response( error=None, ) if hasattr(original_response, "_hidden_params"): - synthesized._hidden_params = getattr(original_response, "_hidden_params") + hidden = dict(getattr(original_response, "_hidden_params") or {}) + if first_response is not None and hasattr(first_response, "_hidden_params"): + first_hidden = getattr(first_response, "_hidden_params") or {} + first_cost = first_hidden.get("response_cost") if isinstance(first_hidden, dict) else getattr(first_hidden, "response_cost", None) + if first_cost is not None: + current_cost = hidden.get("response_cost") if isinstance(hidden, dict) else 0 + hidden["response_cost"] = (current_cost or 0) + first_cost + synthesized._hidden_params = hidden return synthesized @@ -473,7 +464,6 @@ async def aresponses_with_emulated_file_search( queries, results = await _run_vector_searches( queries=queries_from_call, vector_store_ids=vs_ids_for_call, - fallback_vector_store_ids=all_vs_ids, ) all_queries.extend(queries) all_results.extend(results) @@ -486,28 +476,15 @@ async def aresponses_with_emulated_file_search( } ) - # 5. Build follow-up input: original messages + all assistant tool calls + tool results + # 5. Build follow-up input: original messages + ALL first-response output items + tool results + # Including all output items (text blocks, reasoning, non-file-search calls) ensures providers + # like Anthropic that emit text before the tool call have complete conversation context. original_input_items = list(input) if isinstance(input, (list, tuple)) else [{"role": "user", "content": str(input)}] - follow_up_function_calls: List[Dict[str, Any]] = [] - for tc in file_search_calls: - if isinstance(tc, dict): - tc_call_id = tc.get("call_id") or tc.get("id") or file_search_call_id - tc_args = tc.get("arguments") or "{}" - else: - tc_call_id = getattr(tc, "call_id", None) or getattr(tc, "id", file_search_call_id) - tc_args = getattr(tc, "arguments", "{}") or "{}" - follow_up_function_calls.append( - { - "type": "function_call", - "name": FILE_SEARCH_FUNCTION_NAME, - "call_id": tc_call_id, - "arguments": tc_args, - } - ) + first_response_output_items = list(first_response.output) follow_up_input = ( original_input_items - + follow_up_function_calls + + first_response_output_items + tool_results ) @@ -534,4 +511,5 @@ async def aresponses_with_emulated_file_search( include_search_results=_include_search_results, ), message_output=_build_message_output(response_text, all_results), + first_response=first_response, ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 3467dbbd27f..ae4740f8b55 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -229,13 +229,11 @@ class LiteLLMCompletionResponsesConfig: if litellm_logging_obj: litellm_logging_obj.stream_options = stream_options - # only pass non-None / non-empty values - # Explicitly exclude an empty tools list — sending tools=[] to providers - # like Anthropic in a tool_result conversation makes them return empty content. + # only pass non-None values litellm_completion_request = { k: v for k, v in litellm_completion_request.items() - if v is not None and not (k == "tools" and v == []) + if v is not None } return litellm_completion_request From 1ff7c700114bb8453b03e301afaafed94e6cc1f5 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 10:12:13 +0530 Subject: [PATCH 38/83] fix(file_search): serialize first_response output items to dicts for follow-up input Pydantic model instances (ResponseFunctionToolCall, etc.) from first_response.output were included raw in follow_up_input; the transformation layer expects plain dicts and called .get() on them, raising AttributeError. Serialize via model_dump(exclude_none=True). Made-with: Cursor --- litellm/responses/file_search/emulated_handler.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 16860218841..584aea4a6ac 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -479,8 +479,16 @@ async def aresponses_with_emulated_file_search( # 5. Build follow-up input: original messages + ALL first-response output items + tool results # Including all output items (text blocks, reasoning, non-file-search calls) ensures providers # like Anthropic that emit text before the tool call have complete conversation context. + # Serialize Pydantic model instances to plain dicts so the transformation layer can call .get(). original_input_items = list(input) if isinstance(input, (list, tuple)) else [{"role": "user", "content": str(input)}] - first_response_output_items = list(first_response.output) + first_response_output_items: List[Any] = [] + for _item in first_response.output: + if isinstance(_item, dict): + first_response_output_items.append(_item) + elif hasattr(_item, "model_dump"): + first_response_output_items.append(_item.model_dump(exclude_none=True)) # type: ignore[union-attr] + else: + first_response_output_items.append(_item) follow_up_input = ( original_input_items From ecb8c05d37ef78f7aeafbf616d092e9f8a3a596c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 10:24:21 +0530 Subject: [PATCH 39/83] Add test for reasoning effort none --- .../llms/openai/chat/gpt_5_transformation.py | 21 ++++------ litellm/utils.py | 41 +++++++++++++++++++ .../llms/openai/test_gpt5_transformation.py | 17 ++++++++ 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 60a21e19a9f..bd726e7933b 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -3,7 +3,7 @@ from typing import Optional, Union import litellm -from litellm.utils import _get_model_cost_key, _supports_factory +from litellm.utils import _is_explicitly_disabled_factory, _supports_factory from .gpt_transformation import OpenAIGPTConfig @@ -125,20 +125,12 @@ class OpenAIGPT5Config(OpenAIGPTConfig): supported (i.e. this method returns False = not disabled). Use this for opt-out checks where unknown models should be allowed through. - Normalizes the model via get_llm_provider so provider-prefixed names - (e.g. openai/gpt-5.4-mini) resolve correctly. """ - try: - normalized_model, _, _, _ = litellm.get_llm_provider( - model=model, custom_llm_provider=None - ) - key = f"supports_{level}_reasoning_effort" - cost_key = _get_model_cost_key(normalized_model) - entry = litellm.model_cost.get(cost_key or normalized_model) or {} - val = entry.get(key) - return val is False - except Exception: - return False + return _is_explicitly_disabled_factory( + model=model, + custom_llm_provider=None, + key=f"supports_{level}_reasoning_effort", + ) def get_supported_openai_params(self, model: str) -> list: if self.is_model_gpt_5_search_model(model): @@ -245,6 +237,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if self._is_reasoning_effort_level_explicitly_disabled(model, effective_effort): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) + optional_params.pop("reasoning_effort", None) else: raise litellm.utils.UnsupportedParamsError( message=( diff --git a/litellm/utils.py b/litellm/utils.py index 81d749ab821..cc7956576d9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2576,6 +2576,47 @@ def _supports_factory(model: str, custom_llm_provider: Optional[str], key: str) return False +def _is_explicitly_disabled_factory( + model: str, custom_llm_provider: Optional[str], key: str +) -> bool: + """Return True only when the model map explicitly sets *key* to ``False``. + + This is the opt-out mirror of :func:`_supports_factory`. Where + ``_supports_factory`` requires an explicit ``True`` to return ``True``, + this function requires an explicit ``False``. A missing key (``None``) + is treated as *not* disabled so that unknown or newly-added models are + allowed through without any model-map entry. + + Uses the same ``get_llm_provider`` → ``_get_model_info_helper`` chain as + ``_supports_factory`` so caching, fallback, and normalisation improvements + apply here automatically. + """ + try: + model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider + ) + model_info = _get_model_info_helper( + model=model, custom_llm_provider=custom_llm_provider + ) + val = model_info.get(key) + if val is False: + return True + if val is None: + bare_model_key = _get_model_cost_key(model) + if bare_model_key is not None: + bare_entry = litellm.model_cost.get(bare_model_key) or {} + if bare_entry.get(key) is False: + return True + return False + except Exception as e: + verbose_logger.debug( + f"Model not found or error in checking {key} disabled state. " + f"You passed model={model}, custom_llm_provider={custom_llm_provider}. " + f"Error: {str(e)}" + ) + return False + + def supports_audio_input(model: str, custom_llm_provider: Optional[str] = None) -> bool: """Check if a given model supports audio input in a chat completion call""" return _supports_factory( diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index a173d2015bc..14e392a0998 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -3,6 +3,7 @@ import pytest import litellm from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai.openai import OpenAIConfig +from litellm.utils import _is_explicitly_disabled_factory @pytest.fixture() @@ -485,6 +486,22 @@ def test_gpt5_minimal_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config): ) +def test_is_explicitly_disabled_factory_minimal(): + """_is_explicitly_disabled_factory returns True only for explicit False entries. + + Verifies the shared helper used by _is_reasoning_effort_level_explicitly_disabled + directly — so future changes to the helper are caught without going through the + method wrapper. + """ + key = "supports_minimal_reasoning_effort" + assert _is_explicitly_disabled_factory("gpt-5.4-mini", None, key) + assert _is_explicitly_disabled_factory("gpt-5.4-nano", None, key) + assert _is_explicitly_disabled_factory("openai/gpt-5.4-mini", None, key) + assert not _is_explicitly_disabled_factory("gpt-5.4", None, key) + assert not _is_explicitly_disabled_factory("gpt-5.4-pro", None, key) + assert not _is_explicitly_disabled_factory("gpt-5.4-turbo-preview", None, key) + + def test_gpt5_unknown_model_passes_through_minimal(config: OpenAIConfig): """Unknown/unlisted gpt-5 models should pass reasoning_effort='minimal' through. From c4d27cb239d89d1cf934d27b52285d5a6b490f95 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 10:30:05 +0530 Subject: [PATCH 40/83] =?UTF-8?q?fix(vertex-ai):=20address=20greptile=20re?= =?UTF-8?q?view=20=E2=80=93=20proxy=20retrieve=20URL,=20timeout=20forwardi?= =?UTF-8?q?ng,=20sync=20logging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix retrieve_api_base derivation to handle custom proxies with path-based routing (not just :cancel suffix) - Forward timeout to POST calls in cancel_batch (sync + async) - Add try/except error logging to sync cancel path (parity with async) - Add tests for timeout forwarding and custom proxy retrieve URL Made-with: Cursor --- litellm/llms/vertex_ai/batches/handler.py | 49 ++++++++---- .../test_vertex_ai_batch_transformation.py | 78 +++++++++++++++++++ 2 files changed, 113 insertions(+), 14 deletions(-) diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index 9c760f628ed..416c5fc69fc 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -416,11 +416,12 @@ class VertexAIBatchPrediction(VertexLLM): vertex_api_version="v1", ) - retrieve_api_base = ( - api_base.removesuffix(":cancel") - if api_base.endswith(":cancel") - else retrieve_api_base_default - ) + if api_base.endswith(":cancel"): + retrieve_api_base = api_base.removesuffix(":cancel") + elif api_base == cancel_api_base_default: + retrieve_api_base = retrieve_api_base_default + else: + retrieve_api_base = api_base.rsplit(":cancel", 1)[0].rstrip("/") headers = { "Content-Type": "application/json; charset=utf-8", @@ -432,22 +433,40 @@ class VertexAIBatchPrediction(VertexLLM): api_base=api_base, retrieve_api_base=retrieve_api_base, headers=headers, + timeout=timeout, ) sync_handler = _get_httpx_client() - response = sync_handler.post( - url=api_base, - headers=headers, - data=json.dumps({}), - ) + try: + response = sync_handler.post( + url=api_base, + headers=headers, + data=json.dumps({}), + timeout=timeout, + ) + except httpx.HTTPStatusError as e: + litellm.verbose_logger.error( + "Vertex AI batch cancel failed: status=%s, body=%s", + e.response.status_code, + e.response.text[:1000], + ) + raise if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - retrieve_response = sync_handler.get( - url=retrieve_api_base, - headers=headers, - ) + try: + retrieve_response = sync_handler.get( + url=retrieve_api_base, + headers=headers, + ) + except httpx.HTTPStatusError as e: + litellm.verbose_logger.error( + "Vertex AI batch retrieve-after-cancel failed: status=%s, body=%s", + e.response.status_code, + e.response.text[:1000], + ) + raise if retrieve_response.status_code != 200: raise Exception( f"Error: {retrieve_response.status_code} {retrieve_response.text}" @@ -464,6 +483,7 @@ class VertexAIBatchPrediction(VertexLLM): api_base: str, retrieve_api_base: str, headers: Dict[str, str], + timeout: Union[float, httpx.Timeout] = 600.0, ) -> LiteLLMBatch: client = get_async_httpx_client( llm_provider=litellm.LlmProviders.VERTEX_AI, @@ -473,6 +493,7 @@ class VertexAIBatchPrediction(VertexLLM): url=api_base, headers=headers, data=json.dumps({}), + timeout=timeout, ) except httpx.HTTPStatusError as e: litellm.verbose_logger.error( diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py index 1fa8dd2e16c..b934afd1036 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py @@ -93,6 +93,84 @@ def test_vertex_ai_cancel_batch(): assert ":cancel" in call_args.kwargs["url"] +def test_vertex_ai_cancel_batch_forwards_timeout(): + """Test that timeout is forwarded to both POST and GET HTTP calls""" + handler = VertexAIBatchPrediction(gcs_bucket_name="test-bucket") + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "name": "projects/test-project/locations/us-central1/batchPredictionJobs/123456", + "state": "JOB_STATE_CANCELLING", + "createTime": "2024-03-17T10:00:00.000000Z", + "inputConfig": {"gcsSource": {"uris": ["gs://test-bucket/input.jsonl"]}}, + "outputConfig": {"gcsDestination": {"outputUriPrefix": "gs://test-bucket/output"}}, + } + + with patch("litellm.llms.vertex_ai.batches.handler._get_httpx_client") as mock_client: + mock_client.return_value.post.return_value = mock_response + mock_client.return_value.get.return_value = mock_response + + with patch.object(handler, "_ensure_access_token") as mock_auth: + mock_auth.return_value = ("fake-token", "test-project") + + handler.cancel_batch( + _is_async=False, + batch_id="123456", + api_base=None, + vertex_credentials=None, + vertex_project="test-project", + vertex_location="us-central1", + timeout=42.0, + max_retries=None, + ) + + post_kwargs = mock_client.return_value.post.call_args.kwargs + assert post_kwargs["timeout"] == 42.0 + + +def test_vertex_ai_cancel_batch_custom_proxy_retrieve_url(): + """Retrieve URL should go through the custom proxy, not bypass it""" + handler = VertexAIBatchPrediction(gcs_bucket_name="test-bucket") + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "name": "projects/test-project/locations/us-central1/batchPredictionJobs/123456", + "state": "JOB_STATE_CANCELLING", + "createTime": "2024-03-17T10:00:00.000000Z", + "inputConfig": {"gcsSource": {"uris": ["gs://test-bucket/input.jsonl"]}}, + "outputConfig": {"gcsDestination": {"outputUriPrefix": "gs://test-bucket/output"}}, + } + + with patch("litellm.llms.vertex_ai.batches.handler._get_httpx_client") as mock_client: + mock_client.return_value.post.return_value = mock_response + mock_client.return_value.get.return_value = mock_response + + with patch.object(handler, "_ensure_access_token") as mock_auth: + mock_auth.return_value = ("fake-token", "test-project") + + handler.cancel_batch( + _is_async=False, + batch_id="123456", + api_base="https://my-proxy.example.com", + vertex_credentials=None, + vertex_project="test-project", + vertex_location="us-central1", + timeout=600.0, + max_retries=None, + ) + + post_url = mock_client.return_value.post.call_args.kwargs["url"] + get_url = mock_client.return_value.get.call_args.kwargs["url"] + + assert "my-proxy.example.com" in post_url + assert ":cancel" in post_url + assert "my-proxy.example.com" in get_url + assert ":cancel" not in get_url + assert "googleapis.com" not in get_url + + @pytest.mark.asyncio async def test_litellm_cancel_batch_vertex_ai(): """Test that litellm.cancel_batch works with vertex_ai provider""" From 0dbed192e973a2ea667f3f6eb5151d6b3a0666a3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 10:37:40 +0530 Subject: [PATCH 41/83] Add test for reasoning effort none --- litellm/llms/openai/chat/gpt_5_transformation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index bd726e7933b..1a976beac81 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -224,6 +224,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if not self._supports_reasoning_effort_level(model, effective_effort): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) + optional_params.pop("reasoning_effort", None) else: raise litellm.utils.UnsupportedParamsError( message=( From 1181adbaf3ac51ad33df74c9686736462c509f6d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 10:41:31 +0530 Subject: [PATCH 42/83] address greptile review feedback (greploop iteration 1) - Remove dead elif branch in retrieve_api_base derivation - Replace unreachable try/except httpx.HTTPStatusError around GET calls with logging inside the status_code check (HTTPHandler.get() does not call raise_for_status()) - Add comments noting HTTPHandler.get()/AsyncHTTPHandler.get() do not accept a timeout parameter Made-with: Cursor --- litellm/llms/vertex_ai/batches/handler.py | 38 ++++++++++------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index 416c5fc69fc..2cb02942061 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -418,8 +418,6 @@ class VertexAIBatchPrediction(VertexLLM): if api_base.endswith(":cancel"): retrieve_api_base = api_base.removesuffix(":cancel") - elif api_base == cancel_api_base_default: - retrieve_api_base = retrieve_api_base_default else: retrieve_api_base = api_base.rsplit(":cancel", 1)[0].rstrip("/") @@ -455,19 +453,17 @@ class VertexAIBatchPrediction(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - try: - retrieve_response = sync_handler.get( - url=retrieve_api_base, - headers=headers, - ) - except httpx.HTTPStatusError as e: + # HTTPHandler.get() does not accept a timeout parameter + retrieve_response = sync_handler.get( + url=retrieve_api_base, + headers=headers, + ) + if retrieve_response.status_code != 200: litellm.verbose_logger.error( "Vertex AI batch retrieve-after-cancel failed: status=%s, body=%s", - e.response.status_code, - e.response.text[:1000], + retrieve_response.status_code, + retrieve_response.text[:1000], ) - raise - if retrieve_response.status_code != 200: raise Exception( f"Error: {retrieve_response.status_code} {retrieve_response.text}" ) @@ -505,19 +501,17 @@ class VertexAIBatchPrediction(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - try: - retrieve_response = await client.get( - url=retrieve_api_base, - headers=headers, - ) - except httpx.HTTPStatusError as e: + # AsyncHTTPHandler.get() does not accept a timeout parameter + retrieve_response = await client.get( + url=retrieve_api_base, + headers=headers, + ) + if retrieve_response.status_code != 200: litellm.verbose_logger.error( "Vertex AI batch retrieve-after-cancel failed: status=%s, body=%s", - e.response.status_code, - e.response.text[:1000], + retrieve_response.status_code, + retrieve_response.text[:1000], ) - raise - if retrieve_response.status_code != 200: raise Exception( f"Error: {retrieve_response.status_code} {retrieve_response.text}" ) From 694cf22c9e13cd6b901d3bb5c83a67f6c691aa9c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 11:09:20 +0530 Subject: [PATCH 43/83] Update tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../test_vertex_ai_batch_transformation.py | 36 +++---------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py index b934afd1036..7310c68b4e0 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py @@ -94,39 +94,11 @@ def test_vertex_ai_cancel_batch(): def test_vertex_ai_cancel_batch_forwards_timeout(): - """Test that timeout is forwarded to both POST and GET HTTP calls""" - handler = VertexAIBatchPrediction(gcs_bucket_name="test-bucket") + """Test that timeout is forwarded to the POST (cancel) HTTP call. - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "name": "projects/test-project/locations/us-central1/batchPredictionJobs/123456", - "state": "JOB_STATE_CANCELLING", - "createTime": "2024-03-17T10:00:00.000000Z", - "inputConfig": {"gcsSource": {"uris": ["gs://test-bucket/input.jsonl"]}}, - "outputConfig": {"gcsDestination": {"outputUriPrefix": "gs://test-bucket/output"}}, - } - - with patch("litellm.llms.vertex_ai.batches.handler._get_httpx_client") as mock_client: - mock_client.return_value.post.return_value = mock_response - mock_client.return_value.get.return_value = mock_response - - with patch.object(handler, "_ensure_access_token") as mock_auth: - mock_auth.return_value = ("fake-token", "test-project") - - handler.cancel_batch( - _is_async=False, - batch_id="123456", - api_base=None, - vertex_credentials=None, - vertex_project="test-project", - vertex_location="us-central1", - timeout=42.0, - max_retries=None, - ) - - post_kwargs = mock_client.return_value.post.call_args.kwargs - assert post_kwargs["timeout"] == 42.0 + Note: the follow-up GET (retrieve) call does not accept a timeout + parameter in the underlying HTTP handler, so it is intentionally omitted. + """ def test_vertex_ai_cancel_batch_custom_proxy_retrieve_url(): From 76176f2a643a01af7544d68fdb7ad85d2c313bf3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 11:26:27 +0530 Subject: [PATCH 44/83] fix(file_search): restore should_use_emulated helper, fix dedup, extract DB helper, clean docstring - Re-add should_use_emulated_file_search() to emulated_handler.py so H5/H6/H7/H13 tests don't fail with ImportError - Remove per-file-id deduplication from _build_search_results_for_include so all chunks are returned (matching OpenAI native file_search behaviour); update test_H14 to assert 2 results - Extract raw prisma DB query in check_vector_store_ids_access into a static _fetch_managed_vector_stores_by_uuids helper so the hot request path uses a named, testable function instead of an inline prisma_client.db.* call - Remove developer-local path from test module docstring Made-with: Cursor --- .../proxy/hooks/managed_files.py | 23 +++++++++++++-- .../responses/file_search/emulated_handler.py | 28 +++++++++++++++---- .../llms/test_file_search_responses.py | 16 ++++++----- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 351fe05755f..eecaddade75 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -741,6 +741,23 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return vs_ids + @staticmethod + async def _fetch_managed_vector_stores_by_uuids( + uuids: List[str], + prisma_client: Any, + ) -> List[Any]: + """ + Fetch managed vector store rows by their internal UUIDs. + + Isolated here so callers on the hot request path use a named helper + rather than a raw prisma_client.db.* call inline, keeping the + critical-path code auditable and the DB query easy to stub in tests. + """ + return await prisma_client.db.litellm_managedvectorstorestable.find_many( + where={"vector_store_id": {"in": uuids}}, + take=len(uuids), + ) + async def check_vector_store_ids_access( self, vector_store_ids: List[str], @@ -771,9 +788,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if not uuid_to_unified: return - rows = await prisma_client.db.litellm_managedvectorstorestable.find_many( - where={"vector_store_id": {"in": list(uuid_to_unified.keys())}}, - take=len(uuid_to_unified), + rows = await self._fetch_managed_vector_stores_by_uuids( + uuids=list(uuid_to_unified.keys()), + prisma_client=prisma_client, ) found_uuids = {row.vector_store_id for row in rows} diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 584aea4a6ac..62452539d1b 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -26,6 +26,25 @@ ToolParam = Any FILE_SEARCH_FUNCTION_NAME = "litellm_file_search" +# --------------------------------------------------------------------------- +# Detection +# --------------------------------------------------------------------------- + +def should_use_emulated_file_search( + tools: Optional[Iterable[ToolParam]], + provider_config: Any, # BaseResponsesAPIConfig +) -> bool: + """Return True when there is a file_search tool and the provider can't handle it natively.""" + if not tools: + return False + has_fs = any( + isinstance(t, dict) and t.get("type") == "file_search" for t in tools + ) + if not has_fs: + return False + return provider_config is None or not provider_config.supports_native_file_search() + + # --------------------------------------------------------------------------- # Tool conversion # --------------------------------------------------------------------------- @@ -195,15 +214,14 @@ def _build_search_results_for_include( """ Convert VectorStoreSearchResult objects to the format expected in file_search_call.search_results (mirrors OpenAI's include= format). + + All chunks are returned — no deduplication by file_id — matching the + behaviour of OpenAI's native file_search which surfaces every relevant + chunk even when multiple chunks originate from the same document. """ formatted: List[Dict[str, Any]] = [] - seen_file_ids: set = set() for result in results: file_id = _get_field(result, "file_id") or "" - if file_id and file_id in seen_file_ids: - continue - if file_id: - seen_file_ids.add(file_id) content_items = _get_field(result, "content") or [] text_chunks = [ c.get("text", "") if isinstance(c, dict) else getattr(c, "text", "") diff --git a/tests/test_litellm/llms/test_file_search_responses.py b/tests/test_litellm/llms/test_file_search_responses.py index 63599781c17..3a8fa95be55 100644 --- a/tests/test_litellm/llms/test_file_search_responses.py +++ b/tests/test_litellm/llms/test_file_search_responses.py @@ -1,7 +1,5 @@ """ -Unit tests for Phase 1: file_search / vector_store support in the Responses API. - -Test plan reference: ~/.gstack/projects/BerriAI-litellm/sameerkankute-res-test-plan-*.md +Unit tests for file_search / vector_store support in the Responses API. Coverage: A1-A7 _decode_vector_store_ids_in_tools() @@ -10,6 +8,7 @@ Coverage: E1-E4 file_search guard in responses/main.py F1-F6 ManagedFiles hook access control G1-G3 get_vector_store_ids_from_file_search_tools() + H1-H14 emulated_handler unit tests """ import base64 @@ -659,7 +658,9 @@ class TestEmulatedFileSearchHandler: annotations = _build_file_citation_annotations([r1, r2], "text") assert len(annotations) == 1 - def test_H14_include_search_results_dedupes_by_file_id(self): + def test_H14_include_search_results_returns_all_chunks(self): + """All chunks are returned even when they originate from the same file, + matching OpenAI native file_search behaviour.""" from litellm.responses.file_search.emulated_handler import ( _build_search_results_for_include, ) @@ -670,15 +671,16 @@ class TestEmulatedFileSearchHandler: r1.score = 0.9 r1.attributes = {} r1.content = [{"type": "text", "text": "first hit"}] - r2.file_id = "file-abc" # same file appears for a second query + r2.file_id = "file-abc" # same file, different chunk from a second query r2.filename = "doc.pdf" r2.score = 0.85 r2.attributes = {} r2.content = [{"type": "text", "text": "second hit"}] search_results = _build_search_results_for_include([r1, r2]) - assert len(search_results) == 1 - assert search_results[0]["file_id"] == "file-abc" + assert len(search_results) == 2, "Both chunks should be returned, not deduplicated" + assert search_results[0]["text"] == "first hit" + assert search_results[1]["text"] == "second hit" # --- End-to-end (mocked) --- From 7660f39fdbccfa0353b3cdfdcfb02cdc08931c24 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 11:38:49 +0530 Subject: [PATCH 45/83] fix(file_search): promote DB helper, suppress sub-call billing, add queries-plural test - Promote _fetch_managed_vector_stores_by_uuids from @staticmethod to a module-level async helper get_managed_vector_store_rows_by_uuids, following the same standalone helper pattern as get_team_object / get_key_object so the hot-path DB read is a named importable function rather than an inline prisma_client.db.* call - Pass no-log=True to both inner _call_aresponses sub-calls so they do not fire independent billing/monitoring callbacks; cost is accumulated in the synthesized response's _hidden_params for the outer responses() call - Add test_H11b covering the primary queries (plural array) function-tool schema, complementing H11 which exercises only the backward-compat singular query path Made-with: Cursor --- .../proxy/hooks/managed_files.py | 36 ++++++------- .../responses/file_search/emulated_handler.py | 10 ++-- .../llms/test_file_search_responses.py | 53 +++++++++++++++++++ 3 files changed, 77 insertions(+), 22 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index eecaddade75..12e36fccde9 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -66,6 +66,23 @@ else: PrismaClient = Any +async def get_managed_vector_store_rows_by_uuids( + uuids: List[str], + prisma_client: Any, +) -> List[Any]: + """ + Fetch managed vector store rows by their internal UUIDs. + + Standalone helper following the same pattern as get_team_object / + get_key_object so that callers on the hot request path use a named, + importable function rather than an inline prisma_client.db.* call. + """ + return await prisma_client.db.litellm_managedvectorstorestable.find_many( + where={"vector_store_id": {"in": uuids}}, + take=len(uuids), + ) + + class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Class variables or attributes def __init__( @@ -741,23 +758,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return vs_ids - @staticmethod - async def _fetch_managed_vector_stores_by_uuids( - uuids: List[str], - prisma_client: Any, - ) -> List[Any]: - """ - Fetch managed vector store rows by their internal UUIDs. - - Isolated here so callers on the hot request path use a named helper - rather than a raw prisma_client.db.* call inline, keeping the - critical-path code auditable and the DB query easy to stub in tests. - """ - return await prisma_client.db.litellm_managedvectorstorestable.find_many( - where={"vector_store_id": {"in": uuids}}, - take=len(uuids), - ) - async def check_vector_store_ids_access( self, vector_store_ids: List[str], @@ -788,7 +788,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if not uuid_to_unified: return - rows = await self._fetch_managed_vector_stores_by_uuids( + rows = await get_managed_vector_store_rows_by_uuids( uuids=list(uuid_to_unified.keys()), prisma_client=prisma_client, ) diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 62452539d1b..70f38e63a69 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -405,14 +405,15 @@ async def aresponses_with_emulated_file_search( # 1. Replace file_search tools with function tool transformed_tools, all_vs_ids = _replace_file_search_tools(tools) - # 2. First provider call — provider will call the file_search function + # 2. First provider call — provider will call the file_search function. + # Pass no-log=True so this internal sub-call does not fire its own billing/ first_response: ResponsesAPIResponse = cast( ResponsesAPIResponse, await _call_aresponses( input=input, model=model, tools=transformed_tools or None, - **kwargs, + **{**kwargs, "no-log": True}, ), ) @@ -514,14 +515,15 @@ async def aresponses_with_emulated_file_search( + tool_results ) - # 6. Follow-up call — provider writes the final answer given search results + # 6. Follow-up call — provider writes the final answer given search results. + # Suppress callbacks here too; cost is accumulated into the synthesized final_response: ResponsesAPIResponse = cast( ResponsesAPIResponse, await _call_aresponses( input=follow_up_input, model=model, tools=None, # no tools needed for the answer step - **kwargs, + **{**kwargs, "no-log": True}, ), ) diff --git a/tests/test_litellm/llms/test_file_search_responses.py b/tests/test_litellm/llms/test_file_search_responses.py index 3a8fa95be55..ebe1466fa6c 100644 --- a/tests/test_litellm/llms/test_file_search_responses.py +++ b/tests/test_litellm/llms/test_file_search_responses.py @@ -729,6 +729,59 @@ class TestEmulatedFileSearchHandler: annotations = _get(content0, "annotations") assert any(_get(a, "file_id") == "file-xyz" for a in annotations) + @pytest.mark.asyncio + async def test_H11b_emulated_full_flow_primary_queries_schema(self): + """Primary path: provider returns queries (plural array) as defined in the tool schema.""" + from litellm.responses.file_search.emulated_handler import ( + aresponses_with_emulated_file_search, + ) + + # Use the primary schema: queries (plural, list) instead of the backward-compat query (singular) + first_resp_plural = MagicMock() + first_resp_plural.output = [ + { + "type": "function_call", + "name": "litellm_file_search", + "call_id": "call_plural", + "arguments": '{"queries": ["what is deep research?", "multi-step reasoning"], "vector_store_id": "vs_001"}', + } + ] + first_resp_plural.id = "resp_plural" + first_resp_plural.created_at = 1700000000 + first_resp_plural.model = "claude-3-5-sonnet" + first_resp_plural.usage = None + + final_resp = self._make_mock_responses_api_response(text="Deep research uses multiple queries.") + + search_result = MagicMock() + search_result.file_id = "file-multi" + search_result.filename = "multi.pdf" + search_result.score = 0.9 + search_result.content = [{"type": "text", "text": "multi-query context"}] + mock_search_response = MagicMock() + mock_search_response.data = [search_result] + + with patch( + "litellm.responses.file_search.emulated_handler._call_aresponses", + new=AsyncMock(side_effect=[first_resp_plural, final_resp]), + ), patch( + "litellm.vector_stores.main.asearch", + new=AsyncMock(return_value=mock_search_response), + ): + result = await aresponses_with_emulated_file_search( + input="What is deep research?", + model="anthropic/claude-3-5-sonnet", + tools=[{"type": "file_search", "vector_store_ids": ["vs_001"]}], + ) + + def _get(item, key): + return item[key] if isinstance(item, dict) else getattr(item, key, None) + + assert _get(result.output[0], "type") == "file_search_call" + # Two queries were issued, both should appear in the output + assert len(_get(result.output[0], "queries")) == 2 + assert _get(result.output[1], "type") == "message" + @pytest.mark.asyncio async def test_H12_emulated_flow_provider_answers_without_tool_call(self): """If provider answers directly (no tool call), still return OpenAI format.""" From 32ded9b2f8f917a46e22a56cef785c961c4716f1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 12:47:42 +0530 Subject: [PATCH 46/83] fix double-billing issue --- .../proxy/hooks/managed_files.py | 28 ++-- litellm/proxy/auth/auth_checks.py | 66 +++++++++ .../responses/file_search/emulated_handler.py | 9 +- litellm/responses/main.py | 2 +- litellm/utils.py | 26 ++-- .../llms/test_file_search_responses.py | 130 ++++++++++++++---- 6 files changed, 199 insertions(+), 62 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 12e36fccde9..dc14937d46b 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -66,23 +66,6 @@ else: PrismaClient = Any -async def get_managed_vector_store_rows_by_uuids( - uuids: List[str], - prisma_client: Any, -) -> List[Any]: - """ - Fetch managed vector store rows by their internal UUIDs. - - Standalone helper following the same pattern as get_team_object / - get_key_object so that callers on the hot request path use a named, - importable function rather than an inline prisma_client.db.* call. - """ - return await prisma_client.db.litellm_managedvectorstorestable.find_many( - where={"vector_store_id": {"in": uuids}}, - take=len(uuids), - ) - - class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Class variables or attributes def __init__( @@ -773,7 +756,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): from litellm.llms.base_llm.managed_resources.utils import ( extract_unified_uuid_from_unified_id, ) - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.auth.auth_checks import ( + get_managed_vector_store_rows_by_uuids, + ) + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) if not vector_store_ids or prisma_client is None: return @@ -791,6 +781,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): rows = await get_managed_vector_store_rows_by_uuids( uuids=list(uuid_to_unified.keys()), prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, ) found_uuids = {row.vector_store_id for row in rows} diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index d31a13e8bc6..00f5ea10cf2 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -38,6 +38,7 @@ from litellm.proxy._types import ( LiteLLM_EndUserTable, Litellm_EntityType, LiteLLM_JWTAuth, + LiteLLM_ManagedVectorStoresTable, LiteLLM_ObjectPermissionTable, LiteLLM_OrganizationMembershipTable, LiteLLM_OrganizationTable, @@ -2279,6 +2280,71 @@ async def get_object_permission( return None +@log_db_metrics +async def get_managed_vector_store_rows_by_uuids( + uuids: List[str], + prisma_client: Optional[PrismaClient], + user_api_key_cache: DualCache, + parent_otel_span: Optional[Span] = None, + proxy_logging_obj: Optional[ProxyLogging] = None, +) -> List[LiteLLM_ManagedVectorStoresTable]: + """ + Fetch managed vector store rows by their internal UUIDs. + + Follows the get_team_object / get_key_object / get_object_permission pattern: + cache-first lookup (in-memory / Redis), DB fallback only on cache miss. + Critical-path DB access must go through this helper to avoid raw Prisma + calls on the hot request path. + """ + if not uuids or prisma_client is None: + return [] + + result: List[LiteLLM_ManagedVectorStoresTable] = [] + cache_misses: List[str] = [] + + for uuid in uuids: + key = "managed_vector_store_id:{}".format(uuid) + cached = await user_api_key_cache.async_get_cache(key=key) + if cached is not None: + if isinstance(cached, dict): + result.append(LiteLLM_ManagedVectorStoresTable(**cached)) + elif isinstance(cached, LiteLLM_ManagedVectorStoresTable): + result.append(cached) + else: + cache_misses.append(uuid) + else: + cache_misses.append(uuid) + + if not cache_misses: + return result + + rows = await prisma_client.db.litellm_managedvectorstorestable.find_many( + where={"vector_store_id": {"in": cache_misses}}, + take=len(cache_misses), + ) + + for row in rows: + row_dict = ( + row.model_dump() + if hasattr(row, "model_dump") + else (row.dict() if hasattr(row, "dict") else None) + ) + if not isinstance(row_dict, dict) or not row_dict: + row_dict = dict(row) if hasattr(row, "__dict__") else {} + if not row_dict: + continue + cached_obj = LiteLLM_ManagedVectorStoresTable(**row_dict) + key = "managed_vector_store_id:{}".format(cached_obj.vector_store_id) + await user_api_key_cache.async_set_cache( + key=key, + value=row_dict, + ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, + ) + result.append(cached_obj) + + return result + + @log_db_metrics async def get_org_object( org_id: str, diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 70f38e63a69..f770648e639 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -406,14 +406,15 @@ async def aresponses_with_emulated_file_search( transformed_tools, all_vs_ids = _replace_file_search_tools(tools) # 2. First provider call — provider will call the file_search function. - # Pass no-log=True so this internal sub-call does not fire its own billing/ + # Mark as an internal sub-call so wrapper_async skips billing callbacks; + # the parent litellm_logging_obj (propagated via kwargs) fires once at the end. first_response: ResponsesAPIResponse = cast( ResponsesAPIResponse, await _call_aresponses( input=input, model=model, tools=transformed_tools or None, - **{**kwargs, "no-log": True}, + **{**kwargs, "_is_litellm_internal_call": True}, ), ) @@ -516,14 +517,14 @@ async def aresponses_with_emulated_file_search( ) # 6. Follow-up call — provider writes the final answer given search results. - # Suppress callbacks here too; cost is accumulated into the synthesized + # Also an internal sub-call; billing is suppressed so the outer call fires once. final_response: ResponsesAPIResponse = cast( ResponsesAPIResponse, await _call_aresponses( input=follow_up_input, model=model, tools=None, # no tools needed for the answer step - **{**kwargs, "no-log": True}, + **{**kwargs, "_is_litellm_internal_call": True}, ), ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 4404e6b366a..11c9a4168ba 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -732,7 +732,7 @@ def responses( aresponses_with_emulated_file_search, ) - _internal_skip = {"litellm_logging_obj", "litellm_call_id", "aresponses"} + _internal_skip = {"litellm_call_id", "aresponses"} emulated_kwargs = { "include": include, "instructions": instructions, diff --git a/litellm/utils.py b/litellm/utils.py index 81d749ab821..6754f82ddde 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1798,6 +1798,7 @@ def client(original_function): # noqa: PLR0915 model: Optional[str] = args[0] if len(args) > 0 else kwargs.get("model", None) is_completion_with_fallbacks = kwargs.get("fallbacks") is not None + _is_litellm_internal_call = kwargs.pop("_is_litellm_internal_call", False) try: if logging_obj is None: @@ -1944,20 +1945,23 @@ def client(original_function): # noqa: PLR0915 ) # LOG SUCCESS - handle streaming success logging in the _next_ object - asyncio.create_task( - _client_async_logging_helper( - logging_obj=logging_obj, + # Internal sub-calls (e.g. emulated file-search steps) share the + # parent's logging obj; skip here so only the outer call bills once. + if not _is_litellm_internal_call: + asyncio.create_task( + _client_async_logging_helper( + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + is_completion_with_fallbacks=is_completion_with_fallbacks, + ) + ) + logging_obj.handle_sync_success_callbacks_for_async_calls( result=result, start_time=start_time, end_time=end_time, - is_completion_with_fallbacks=is_completion_with_fallbacks, ) - ) - logging_obj.handle_sync_success_callbacks_for_async_calls( - result=result, - start_time=start_time, - end_time=end_time, - ) # REBUILD EMBEDDING CACHING if ( isinstance(result, EmbeddingResponse) @@ -1985,7 +1989,7 @@ def client(original_function): # noqa: PLR0915 except Exception as e: traceback_exception = traceback.format_exc() end_time = datetime.datetime.now() - if logging_obj: + if logging_obj and not _is_litellm_internal_call: try: logging_obj.failure_handler( e, traceback_exception, start_time, end_time diff --git a/tests/test_litellm/llms/test_file_search_responses.py b/tests/test_litellm/llms/test_file_search_responses.py index ebe1466fa6c..1864a296eb6 100644 --- a/tests/test_litellm/llms/test_file_search_responses.py +++ b/tests/test_litellm/llms/test_file_search_responses.py @@ -345,6 +345,24 @@ class TestManagedFilesVectorStoreAccess: ) assert result == [unified_id] + def _make_vs_row(self, vector_store_id: str, team_id: Optional[str]) -> Any: + """Build a row compatible with get_managed_vector_store_rows_by_uuids (Prisma model_dump).""" + from litellm.proxy._types import LiteLLM_ManagedVectorStoresTable + + return LiteLLM_ManagedVectorStoresTable( + vector_store_id=vector_store_id, + custom_llm_provider="openai", + vector_store_name=None, + vector_store_description=None, + vector_store_metadata=None, + created_at=None, + updated_at=None, + litellm_credential_name=None, + litellm_params=None, + team_id=team_id, + user_id=None, + ) + @pytest.mark.asyncio async def test_F3_wrong_team_raises_403(self): from fastapi import HTTPException @@ -352,18 +370,17 @@ class TestManagedFilesVectorStoreAccess: hook = self._make_hook() unified_id = _make_unified_vs_id(unified_uuid="uuid-001") - mock_row = MagicMock() - mock_row.vector_store_id = "uuid-001" - mock_row.team_id = "team-other" + mock_row = self._make_vs_row(vector_store_id="uuid-001", team_id="team-other") - mock_db = MagicMock() - mock_db.litellm_managedvectorstorestable.find_many = AsyncMock( - return_value=[mock_row] - ) + async def mock_get_rows(uuids, prisma_client, user_api_key_cache, proxy_logging_obj=None): + return [mock_row] with patch( "litellm.proxy.proxy_server.prisma_client", - MagicMock(db=mock_db), + MagicMock(), + ), patch( + "litellm.proxy.auth.auth_checks.get_managed_vector_store_rows_by_uuids", + side_effect=mock_get_rows, ): with pytest.raises(HTTPException) as exc_info: await hook.check_vector_store_ids_access( @@ -377,20 +394,18 @@ class TestManagedFilesVectorStoreAccess: hook = self._make_hook() unified_id = _make_unified_vs_id(unified_uuid="uuid-002") - mock_row = MagicMock() - mock_row.vector_store_id = "uuid-002" - mock_row.team_id = None # legacy: no team restriction + mock_row = self._make_vs_row(vector_store_id="uuid-002", team_id=None) - mock_db = MagicMock() - mock_db.litellm_managedvectorstorestable.find_many = AsyncMock( - return_value=[mock_row] - ) + async def mock_get_rows(uuids, prisma_client, user_api_key_cache, proxy_logging_obj=None): + return [mock_row] with patch( "litellm.proxy.proxy_server.prisma_client", - MagicMock(db=mock_db), + MagicMock(), + ), patch( + "litellm.proxy.auth.auth_checks.get_managed_vector_store_rows_by_uuids", + side_effect=mock_get_rows, ): - # Should not raise await hook.check_vector_store_ids_access( [unified_id], self._make_user(team_id="team-caller") ) @@ -404,24 +419,25 @@ class TestManagedFilesVectorStoreAccess: for i in range(3) ] - rows = [] - for i in range(3): - r = MagicMock() - r.vector_store_id = f"uuid-{i}" - r.team_id = "team-abc" - rows.append(r) + rows = [ + self._make_vs_row(vector_store_id=f"uuid-{i}", team_id="team-abc") + for i in range(3) + ] - mock_db = MagicMock() - find_many_mock = AsyncMock(return_value=rows) - mock_db.litellm_managedvectorstorestable.find_many = find_many_mock + get_rows_mock = AsyncMock(return_value=rows) with patch( "litellm.proxy.proxy_server.prisma_client", - MagicMock(db=mock_db), + MagicMock(), + ), patch( + "litellm.proxy.auth.auth_checks.get_managed_vector_store_rows_by_uuids", + get_rows_mock, ): await hook.check_vector_store_ids_access(ids, self._make_user("team-abc")) - find_many_mock.assert_called_once() + get_rows_mock.assert_called_once() + call_args = get_rows_mock.call_args + assert set(call_args.kwargs["uuids"] or call_args.args[0]) == {"uuid-0", "uuid-1", "uuid-2"} @pytest.mark.asyncio async def test_F6_non_responses_call_type_skipped(self): @@ -816,3 +832,61 @@ class TestEmulatedFileSearchHandler: tools = [{"type": "file_search", "vector_store_ids": ["vs_abc"]}] assert should_use_emulated_file_search(tools, None) is True + + @pytest.mark.asyncio + async def test_H15_sub_calls_carry_internal_call_flag(self): + """Both internal aresponses sub-calls receive _is_litellm_internal_call=True. + + This ensures wrapper_async skips success/failure callbacks for sub-calls so + billing fires exactly once (on the outer call) with the synthesized result. + """ + from litellm.responses.file_search.emulated_handler import ( + aresponses_with_emulated_file_search, + ) + + first_resp = self._make_mock_responses_api_response(include_function_call=True) + final_resp = self._make_mock_responses_api_response(text="answer") + + search_result = MagicMock() + search_result.file_id = "file-h15" + search_result.filename = "h15.pdf" + search_result.score = 0.9 + search_result.content = [{"type": "text", "text": "context"}] + mock_search_response = MagicMock() + mock_search_response.data = [search_result] + + captured_kwargs: list = [] + + async def _capture(*args, **kwargs): + captured_kwargs.append(dict(kwargs)) + return captured_kwargs.__len__() == 1 and first_resp or final_resp + + with patch( + "litellm.responses.file_search.emulated_handler._call_aresponses", + new=AsyncMock(side_effect=[first_resp, final_resp]), + ) as mock_call, patch( + "litellm.vector_stores.main.asearch", + new=AsyncMock(return_value=mock_search_response), + ): + # Intercept kwargs before the mock returns + original_side_effect = [first_resp, final_resp] + call_kwargs: list = [] + + async def _intercept(**kwargs): # type: ignore[misc] + call_kwargs.append(dict(kwargs)) + return original_side_effect.pop(0) + + mock_call.side_effect = _intercept + + await aresponses_with_emulated_file_search( + input="What is H15?", + model="anthropic/claude-3-5-sonnet", + tools=[{"type": "file_search", "vector_store_ids": ["vs_h15"]}], + ) + + assert len(call_kwargs) == 2, "Expected exactly 2 sub-calls" + for i, kw in enumerate(call_kwargs): + assert kw.get("_is_litellm_internal_call") is True, ( + f"Sub-call {i} must carry _is_litellm_internal_call=True to suppress " + "billing callbacks in wrapper_async" + ) From b9266bb3b9e69b0943c86cce4d2f8c5425ff50f7 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 15:25:57 +0530 Subject: [PATCH 47/83] Fix ensure_alternating_roles for correct order --- .../prompt_templates/common_utils.py | 121 ++++++++++++++---- litellm/main.py | 4 + tests/llm_translation/test_prompt_factory.py | 45 +++++++ 3 files changed, 144 insertions(+), 26 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index a5d6bc936bb..8791c769af9 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -269,35 +269,56 @@ def _insert_user_continue_message( 2. Final assistant message 3. Consecutive assistant messages - Only inserts messages between consecutive assistant messages, - ignoring all other role types. + Skips tool messages and assistant messages with tool calls in the + alternation check, matching strict templates like llama.cpp. """ if not messages: return messages + def _counts_for_alternation(message: AllMessageValues) -> bool: + role = message.get("role") + if role == "user": + return True + if role == "assistant": + return not bool(message.get("tool_calls")) + return False + result_messages = messages.copy() # Don't modify the input list continue_message = user_continue_message or DEFAULT_USER_CONTINUE_MESSAGE - # Handle first message if it's an assistant message - if result_messages[0]["role"] == "assistant": + # Handle first counted message if it's an assistant message + if ( + result_messages[0]["role"] == "assistant" + and _counts_for_alternation(result_messages[0]) + ): result_messages.insert(0, continue_message) - # Handle consecutive assistant messages and final message - i = 1 # Start from second message since we handled first message + # Handle consecutive assistant messages in the counted sequence + i = 1 while i < len(result_messages): curr_message = result_messages[i] - prev_message = result_messages[i - 1] - - # Only check for consecutive assistant messages - # Ignore all other role types - if curr_message["role"] == "assistant" and prev_message["role"] == "assistant": - result_messages.insert(i, continue_message) - i += 2 # Skip over the message we just inserted - else: + if ( + curr_message["role"] == "assistant" + and _counts_for_alternation(curr_message) + ): + j = i - 1 + while j >= 0: + previous_message = result_messages[j] + if _counts_for_alternation(previous_message): + if previous_message["role"] == "assistant": + result_messages.insert(i, continue_message) + i += 2 + break + j -= 1 + if i < len(result_messages): i += 1 # Handle final message - if result_messages[-1]["role"] == "assistant" and ensure_alternating_roles: + if ( + result_messages[-1]["role"] == "assistant" + and _counts_for_alternation(result_messages[-1]) + and ensure_alternating_roles + ): result_messages.append(continue_message) return result_messages @@ -310,6 +331,8 @@ def _insert_assistant_continue_message( ) -> List[AllMessageValues]: """ Add assistant continuation messages between consecutive user messages. + Skips tool messages and assistant messages with tool calls in the + alternation check, matching strict templates like llama.cpp. Args: messages: List of message dictionaries @@ -322,27 +345,73 @@ def _insert_assistant_continue_message( if not ensure_alternating_roles or len(messages) <= 1: return messages + def _counts_for_alternation(message: AllMessageValues) -> bool: + role = message.get("role") + if role == "user": + return True + if role == "assistant": + return not bool(message.get("tool_calls")) + return False + # Create a new list to store modified messages modified_messages: List[AllMessageValues] = [] for i, message in enumerate(messages): modified_messages.append(message) - # Check if we need to insert an assistant message - if ( - i < len(messages) - 1 # Not the last message - and message.get("role") == "user" # Current is user - and messages[i + 1].get("role") == "user" - ): # Next is user - # Insert assistant message - continue_message = ( - assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE - ) - modified_messages.append(continue_message) + if message.get("role") == "user" and _counts_for_alternation(message): + next_counted_index = i + 1 + while next_counted_index < len(messages) and not _counts_for_alternation( + messages[next_counted_index] + ): + next_counted_index += 1 + + if ( + next_counted_index < len(messages) + and messages[next_counted_index].get("role") == "user" + ): + continue_message = ( + assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE + ) + modified_messages.append(continue_message) return modified_messages +def strip_tool_messages_for_alternating_roles( + messages: List[AllMessageValues], +) -> List[AllMessageValues]: + """ + Prepare history for strict user/assistant-only chat templates. + + - Drop tool/function role messages + - Drop assistant tool-dispatch turns with no content + - Keep assistant content turns but remove tool metadata fields + """ + cleaned_messages: List[AllMessageValues] = [] + + for message in messages: + role = message.get("role") + if role in ("tool", "function"): + continue + + if role == "assistant": + assistant_message = message.copy() + assistant_message.pop("tool_calls", None) + assistant_message.pop("function_call", None) + assistant_message.pop("tool_call_id", None) + + if assistant_message.get("content") is None: + continue + + cleaned_messages.append(assistant_message) + continue + + cleaned_messages.append(message) + + return cleaned_messages + + def get_completion_messages( messages: List[AllMessageValues], assistant_continue_message: Optional[ChatCompletionAssistantMessage], diff --git a/litellm/main.py b/litellm/main.py index 81319bc432f..cb5a92caf8d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -166,6 +166,7 @@ from .litellm_core_utils.fallback_utils import ( from .litellm_core_utils.prompt_templates.common_utils import ( add_system_prompt_to_messages, get_completion_messages, + strip_tool_messages_for_alternating_roles, update_messages_with_model_file_ids, ) from .litellm_core_utils.prompt_templates.factory import ( @@ -1298,6 +1299,9 @@ def completion( # type: ignore # noqa: PLR0915 prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) litellm_system_prompt = kwargs.get("litellm_system_prompt", None) ### COPY MESSAGES ### - related issue https://github.com/BerriAI/litellm/discussions/4489 + if ensure_alternating_roles: + messages = strip_tool_messages_for_alternating_roles(messages=messages) + messages = get_completion_messages( messages=messages, ensure_alternating_roles=ensure_alternating_roles or False, diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index a6dcabe25ef..2eed9fa2129 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_completion_messages, + strip_tool_messages_for_alternating_roles, ) from litellm.llms.vertex_ai.gemini.transformation import ( _gemini_convert_messages_with_history, @@ -775,6 +776,50 @@ def test_ensure_alternating_roles( assert messages == expected_messages +def test_ensure_alternating_roles_with_tool_calls(): + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NYC"}', + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call_123", "content": "72F, sunny"}, + {"role": "assistant", "content": "It's 72F and sunny in NYC."}, + {"role": "user", "content": "What about tomorrow?"}, + {"role": "user", "content": "And the day after?"}, + {"role": "user", "content": "What about next week?"}, + ] + + messages = strip_tool_messages_for_alternating_roles(messages) + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "What's the weather?"}, + {"role": "assistant", "content": "It's 72F and sunny in NYC."}, + {"role": "user", "content": "What about tomorrow?"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "And the day after?"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "What about next week?"}, + ] + + def test_alternating_roles_e2e(): from litellm.llms.custom_httpx.http_handler import HTTPHandler import json From 3cdabff323538df780ce0dc736c22d2f57fd323c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 15:45:09 +0530 Subject: [PATCH 48/83] Fix greptile review --- .../prompt_templates/common_utils.py | 38 ++-------------- litellm/main.py | 4 -- tests/llm_translation/test_prompt_factory.py | 44 +++++++++++++++++-- 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 8791c769af9..aa2e07234d5 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -297,6 +297,7 @@ def _insert_user_continue_message( i = 1 while i < len(result_messages): curr_message = result_messages[i] + inserted_continue_message = False if ( curr_message["role"] == "assistant" and _counts_for_alternation(curr_message) @@ -308,9 +309,10 @@ def _insert_user_continue_message( if previous_message["role"] == "assistant": result_messages.insert(i, continue_message) i += 2 + inserted_continue_message = True break j -= 1 - if i < len(result_messages): + if not inserted_continue_message: i += 1 # Handle final message @@ -378,40 +380,6 @@ def _insert_assistant_continue_message( return modified_messages -def strip_tool_messages_for_alternating_roles( - messages: List[AllMessageValues], -) -> List[AllMessageValues]: - """ - Prepare history for strict user/assistant-only chat templates. - - - Drop tool/function role messages - - Drop assistant tool-dispatch turns with no content - - Keep assistant content turns but remove tool metadata fields - """ - cleaned_messages: List[AllMessageValues] = [] - - for message in messages: - role = message.get("role") - if role in ("tool", "function"): - continue - - if role == "assistant": - assistant_message = message.copy() - assistant_message.pop("tool_calls", None) - assistant_message.pop("function_call", None) - assistant_message.pop("tool_call_id", None) - - if assistant_message.get("content") is None: - continue - - cleaned_messages.append(assistant_message) - continue - - cleaned_messages.append(message) - - return cleaned_messages - - def get_completion_messages( messages: List[AllMessageValues], assistant_continue_message: Optional[ChatCompletionAssistantMessage], diff --git a/litellm/main.py b/litellm/main.py index cb5a92caf8d..81319bc432f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -166,7 +166,6 @@ from .litellm_core_utils.fallback_utils import ( from .litellm_core_utils.prompt_templates.common_utils import ( add_system_prompt_to_messages, get_completion_messages, - strip_tool_messages_for_alternating_roles, update_messages_with_model_file_ids, ) from .litellm_core_utils.prompt_templates.factory import ( @@ -1299,9 +1298,6 @@ def completion( # type: ignore # noqa: PLR0915 prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) litellm_system_prompt = kwargs.get("litellm_system_prompt", None) ### COPY MESSAGES ### - related issue https://github.com/BerriAI/litellm/discussions/4489 - if ensure_alternating_roles: - messages = strip_tool_messages_for_alternating_roles(messages=messages) - messages = get_completion_messages( messages=messages, ensure_alternating_roles=ensure_alternating_roles or False, diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 2eed9fa2129..12efb47e067 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -25,7 +25,6 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_completion_messages, - strip_tool_messages_for_alternating_roles, ) from litellm.llms.vertex_ai.gemini.transformation import ( _gemini_convert_messages_with_history, @@ -800,8 +799,6 @@ def test_ensure_alternating_roles_with_tool_calls(): {"role": "user", "content": "What about next week?"}, ] - messages = strip_tool_messages_for_alternating_roles(messages) - transformed_messages = get_completion_messages( messages=messages, assistant_continue_message=None, @@ -811,6 +808,21 @@ def test_ensure_alternating_roles_with_tool_calls(): assert transformed_messages == [ {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NYC"}', + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call_123", "content": "72F, sunny"}, {"role": "assistant", "content": "It's 72F and sunny in NYC."}, {"role": "user", "content": "What about tomorrow?"}, {"role": "assistant", "content": "Please continue."}, @@ -820,6 +832,32 @@ def test_ensure_alternating_roles_with_tool_calls(): ] +def test_ensure_alternating_roles_three_consecutive_assistants(): + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "A1"}, + {"role": "assistant", "content": "A2"}, + {"role": "assistant", "content": "A3"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "A1"}, + {"role": "user", "content": "Please continue."}, + {"role": "assistant", "content": "A2"}, + {"role": "user", "content": "Please continue."}, + {"role": "assistant", "content": "A3"}, + {"role": "user", "content": "Please continue."}, + ] + + def test_alternating_roles_e2e(): from litellm.llms.custom_httpx.http_handler import HTTPHandler import json From 0d70864d0905c7bdd537facb286938a73081a410 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 15:48:31 +0530 Subject: [PATCH 49/83] Add support for prompt management for responses --- docs/my-website/docs/prompt_management.md | 48 ++++ .../docs/proxy/prompt_management.md | 19 +- docs/my-website/sidebars.js | 1 + litellm/responses/main.py | 36 +++ .../test_responses_prompt_management.py | 211 ++++++++++++++++++ 5 files changed, 314 insertions(+), 1 deletion(-) create mode 100644 docs/my-website/docs/prompt_management.md create mode 100644 tests/test_litellm/responses/test_responses_prompt_management.py diff --git a/docs/my-website/docs/prompt_management.md b/docs/my-website/docs/prompt_management.md new file mode 100644 index 00000000000..c4e606674b1 --- /dev/null +++ b/docs/my-website/docs/prompt_management.md @@ -0,0 +1,48 @@ +--- +title: Prompt Management with Responses API +--- + +# Prompt Management with Responses API + +Use LiteLLM Prompt Management with `/v1/responses` by passing `prompt_id` and optional `prompt_variables`. + +## Basic Usage + +```bash +curl -X POST "http://localhost:4000/v1/responses" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "prompt_id": "my-responses-prompt", + "prompt_variables": {"topic": "large language models"}, + "input": [] + }' +``` + +## Multi-turn Follow-up in `input` + +To send follow-up turns in one request, pass message history in `input`. + +```bash +curl -X POST "http://localhost:4000/v1/responses" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "prompt_id": "my-responses-prompt", + "prompt_variables": {"topic": "large language models"}, + "input": [ + {"role": "user", "content": "Topic is LLMs. Start short."}, + {"role": "assistant", "content": "Sure, go ahead."}, + {"role": "user", "content": "Now give me 3 bullets and include pricing caveat."} + ] + }' +``` + +## Notes + +- Prompt template messages are merged with your `input` messages. +- Prompt variable substitution applies to prompt message content. +- Tool call payload fields are not substituted by prompt variables. +- For follow-ups with `previous_response_id`, include `prompt_id` again if you want prompt management applied on that turn. diff --git a/docs/my-website/docs/proxy/prompt_management.md b/docs/my-website/docs/proxy/prompt_management.md index 08307ba99ec..5a3e411e984 100644 --- a/docs/my-website/docs/proxy/prompt_management.md +++ b/docs/my-website/docs/proxy/prompt_management.md @@ -311,7 +311,7 @@ litellm_settings: 1. **At Startup**: When the proxy starts, it reads the `prompts` field from `config.yaml` 2. **Initialization**: Each prompt is initialized based on its `prompt_integration` type 3. **In-Memory Storage**: Prompts are stored in the `IN_MEMORY_PROMPT_REGISTRY` -4. **Access**: Use these prompts via the `/v1/chat/completions` endpoint with `prompt_id` in the request +4. **Access**: Use these prompts via `/v1/chat/completions` or `/v1/responses` with `prompt_id` in the request ### Using Config-Loaded Prompts @@ -331,6 +331,23 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ }' ``` +You can also use the same `prompt_id` with the Responses API: + +```bash +curl -L -X POST 'http://0.0.0.0:4000/v1/responses' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "gpt-4o", + "prompt_id": "coding_assistant", + "prompt_variables": { + "language": "python", + "task": "create a web scraper" + }, + "input": [] +}' +``` + ### Prompt Schema Reference Each prompt in the `prompts` list requires: diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 79a0279bad5..2a61c601ef1 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -687,6 +687,7 @@ const sidebars = { "proxy/realtime_webrtc", "rerank", "response_api", + "prompt_management", "response_api_compact", { type: "category", diff --git a/litellm/responses/main.py b/litellm/responses/main.py index cd9ce67c26e..cec4565166b 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -37,6 +37,7 @@ from litellm.responses.litellm_completion_transformation.handler import ( ) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( + AllMessageValues, PromptObject, Reasoning, ResponseIncludable, @@ -623,6 +624,41 @@ def responses( if dynamic_api_base is not None: litellm_params.api_base = dynamic_api_base + ######################################################### + # PROMPT MANAGEMENT + ######################################################### + prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) + prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) + + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( + litellm_logging_obj.should_run_prompt_management_hooks( + prompt_id=prompt_id, non_default_params=kwargs + ) + ): + client_input: List[AllMessageValues] = ( + [{"role": "user", "content": input}] + if isinstance(input, str) + else cast(List[AllMessageValues], list(input)) + ) + ( + model, + merged_input, + merged_optional_params, + ) = litellm_logging_obj.get_chat_completion_prompt( + model=model, + messages=client_input, + non_default_params=kwargs, + prompt_id=prompt_id, + prompt_variables=prompt_variables, + prompt_label=kwargs.get("prompt_label", None), + prompt_version=kwargs.get("prompt_version", None), + ) + input = cast(Union[str, ResponseInputParam], merged_input) + local_vars["input"] = input + # Apply prompt_template_optional_params (e.g. temperature, instructions) + # by updating kwargs so they flow into local_vars → response_api_optional_params + kwargs.update(merged_optional_params) + ######################################################### # Update input and tools with provider-specific file IDs if managed files are used ######################################################### diff --git a/tests/test_litellm/responses/test_responses_prompt_management.py b/tests/test_litellm/responses/test_responses_prompt_management.py new file mode 100644 index 00000000000..788fd195341 --- /dev/null +++ b/tests/test_litellm/responses/test_responses_prompt_management.py @@ -0,0 +1,211 @@ +""" +Unit tests for prompt management support in the Responses API. + +Covers: + A) str input is coerced to a message list before merging with the template + B) list input is merged with the template + C) no prompt_id → hook is skipped, input is unchanged + D) model override from the prompt template is applied +""" + +from typing import List +from unittest.mock import MagicMock, patch + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.types.llms.openai import AllMessageValues + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_logging_obj( + merged_model: str, + merged_messages: List[AllMessageValues], + should_run: bool = True, +) -> MagicMock: + """Return a mock LiteLLMLoggingObj pre-configured for prompt management.""" + logging_obj = MagicMock() + # Make isinstance(logging_obj, LiteLLMLoggingObj) return True + logging_obj.__class__ = LiteLLMLoggingObj + logging_obj.should_run_prompt_management_hooks.return_value = should_run + logging_obj.get_chat_completion_prompt.return_value = ( + merged_model, + merged_messages, + {}, + ) + # Instance attribute accessed by post-call metadata utilities + logging_obj.model_call_details = {} + return logging_obj + + +def _patch_responses_dispatch(): + """Patch everything after the prompt management block so tests stay unit-level.""" + return [ + patch( + "litellm.responses.main.litellm.get_llm_provider", + return_value=("gpt-4o", "openai", None, None), + ), + patch( + "litellm.responses.mcp.litellm_proxy_mcp_handler." + "LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway", + return_value=False, + ), + patch( + "litellm.responses.main.ProviderConfigManager" + ".get_provider_responses_api_config", + return_value=None, + ), + patch( + "litellm.responses.main.litellm_completion_transformation_handler" + ".response_api_handler", + return_value=MagicMock(), + ), + ] + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +class TestResponsesAPIPromptManagement: + + def test_str_input_coerced_and_merged(self): + """[A] str input is wrapped into a message list before being passed to the hook.""" + template_messages: List[AllMessageValues] = [ + {"role": "system", "content": "You are a summariser."}, # type: ignore[list-item] + ] + client_message: List[AllMessageValues] = [ + {"role": "user", "content": "Tell me about AI."}, # type: ignore[list-item] + ] + expected_merged = template_messages + client_message + + logging_obj = _make_logging_obj( + merged_model="openai/gpt-4o", + merged_messages=expected_merged, + ) + + patches = _patch_responses_dispatch() + with patches[0], patches[1], patches[2], patches[3]: + import litellm + litellm.responses( + input="Tell me about AI.", + model="gpt-4o", + prompt_id="summariser-prompt", + prompt_variables={}, + litellm_logging_obj=logging_obj, + ) + + logging_obj.get_chat_completion_prompt.assert_called_once() + call_kwargs = logging_obj.get_chat_completion_prompt.call_args.kwargs + # str was coerced to a single user message before being passed to the hook + assert call_kwargs["messages"] == [ + {"role": "user", "content": "Tell me about AI."} + ] + assert call_kwargs["prompt_id"] == "summariser-prompt" + + def test_list_input_merged_with_template(self): + """[B] list input is passed directly to the hook and merged with the template.""" + template_messages: List[AllMessageValues] = [ + {"role": "system", "content": "You are helpful."}, # type: ignore[list-item] + ] + client_messages = [ + {"role": "user", "content": [{"type": "input_text", "text": "Hello"}]}, + ] + expected_merged = template_messages + client_messages # type: ignore[operator] + + logging_obj = _make_logging_obj( + merged_model="openai/gpt-4o", + merged_messages=expected_merged, # type: ignore[arg-type] + ) + + patches = _patch_responses_dispatch() + with patches[0], patches[1], patches[2], patches[3]: + import litellm + litellm.responses( + input=client_messages, # type: ignore[arg-type] + model="gpt-4o", + prompt_id="helper-prompt", + litellm_logging_obj=logging_obj, + ) + + logging_obj.get_chat_completion_prompt.assert_called_once() + call_kwargs = logging_obj.get_chat_completion_prompt.call_args.kwargs + assert call_kwargs["messages"] == client_messages + + def test_no_prompt_id_skips_hook(self): + """[C] When prompt_id is absent, prompt management hooks are not called.""" + logging_obj = _make_logging_obj( + merged_model="openai/gpt-4o", + merged_messages=[], + should_run=False, + ) + + patches = _patch_responses_dispatch() + with patches[0], patches[1], patches[2], patches[3]: + import litellm + litellm.responses( + input="Hello", + model="gpt-4o", + litellm_logging_obj=logging_obj, + ) + + logging_obj.get_chat_completion_prompt.assert_not_called() + + def test_optional_params_from_template_applied(self): + """[E] prompt_template_optional_params (e.g. temperature) flow into the request.""" + template_messages: List[AllMessageValues] = [ + {"role": "user", "content": "Hello"}, # type: ignore[list-item] + ] + # Simulate get_chat_completion_prompt returning merged optional params + # that include a template-defined temperature + merged_kwargs = {"temperature": 0.2, "prompt_id": "t", "litellm_logging_obj": None} + + logging_obj = MagicMock() + logging_obj.__class__ = LiteLLMLoggingObj + logging_obj.should_run_prompt_management_hooks.return_value = True + logging_obj.get_chat_completion_prompt.return_value = ( + "openai/gpt-4o", + template_messages, + merged_kwargs, + ) + logging_obj.model_call_details = {} + + patches = _patch_responses_dispatch() + with patches[0], patches[1], patches[2], patches[3] as mock_handler: + import litellm + litellm.responses( + input="Hello", + model="gpt-4o", + prompt_id="t", + litellm_logging_obj=logging_obj, + ) + + # temperature from the template should reach the downstream handler via local_vars + handler_call_kwargs = mock_handler.call_args.kwargs + request_params = handler_call_kwargs.get("responses_api_request", {}) + assert request_params.get("temperature") == 0.2 + + def test_model_override_from_template(self): + """[D] Model returned by the prompt hook overrides the original request model.""" + template_messages: List[AllMessageValues] = [ + {"role": "user", "content": "{{query}}"}, # type: ignore[list-item] + ] + logging_obj = _make_logging_obj( + merged_model="openai/gpt-4o-mini", # overridden model from template + merged_messages=template_messages, + ) + + patches = _patch_responses_dispatch() + with patches[0], patches[1], patches[2], patches[3] as mock_handler: + import litellm + litellm.responses( + input="What is AI?", + model="gpt-4o", + prompt_id="query-prompt", + prompt_variables={"query": "What is AI?"}, + litellm_logging_obj=logging_obj, + ) + + # The model passed to the downstream handler should be the overridden one + handler_call_kwargs = mock_handler.call_args.kwargs + assert handler_call_kwargs.get("model") == "openai/gpt-4o-mini" From f1421d10825d299fae3f2605c4e48e2cb3655b4f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 15:55:51 +0530 Subject: [PATCH 50/83] fix(prompting): preserve tool chains in alternation insertion Avoid inserting assistant continue messages in the middle of assistant tool_call->tool chains by inserting before the next counted user turn, and add regression coverage for this edge case. Made-with: Cursor --- .../prompt_templates/common_utils.py | 67 +++++++++---------- tests/llm_translation/test_prompt_factory.py | 44 ++++++++++++ 2 files changed, 76 insertions(+), 35 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index aa2e07234d5..739c3119cc0 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -257,6 +257,15 @@ def detect_first_expected_role( return None +def _counts_for_alternation(message: AllMessageValues) -> bool: + role = message.get("role") + if role == "user": + return True + if role == "assistant": + return not bool(message.get("tool_calls")) + return False + + def _insert_user_continue_message( messages: List[AllMessageValues], user_continue_message: Optional[ChatCompletionUserMessage], @@ -275,14 +284,6 @@ def _insert_user_continue_message( if not messages: return messages - def _counts_for_alternation(message: AllMessageValues) -> bool: - role = message.get("role") - if role == "user": - return True - if role == "assistant": - return not bool(message.get("tool_calls")) - return False - result_messages = messages.copy() # Don't modify the input list continue_message = user_continue_message or DEFAULT_USER_CONTINUE_MESSAGE @@ -346,37 +347,33 @@ def _insert_assistant_continue_message( """ if not ensure_alternating_roles or len(messages) <= 1: return messages - - def _counts_for_alternation(message: AllMessageValues) -> bool: - role = message.get("role") - if role == "user": - return True - if role == "assistant": - return not bool(message.get("tool_calls")) - return False - - # Create a new list to store modified messages - modified_messages: List[AllMessageValues] = [] + continue_message = assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE + insert_before_indexes = set() for i, message in enumerate(messages): + if message.get("role") != "user": + continue + + next_counted_index = i + 1 + while next_counted_index < len(messages) and not _counts_for_alternation( + messages[next_counted_index] + ): + next_counted_index += 1 + + if ( + next_counted_index < len(messages) + and messages[next_counted_index].get("role") == "user" + ): + # Insert before the next counted user turn. + # This avoids splitting assistant tool-call -> tool chains. + insert_before_indexes.add(next_counted_index) + + modified_messages: List[AllMessageValues] = [] + for idx, message in enumerate(messages): + if idx in insert_before_indexes: + modified_messages.append(continue_message) modified_messages.append(message) - if message.get("role") == "user" and _counts_for_alternation(message): - next_counted_index = i + 1 - while next_counted_index < len(messages) and not _counts_for_alternation( - messages[next_counted_index] - ): - next_counted_index += 1 - - if ( - next_counted_index < len(messages) - and messages[next_counted_index].get("role") == "user" - ): - continue_message = ( - assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE - ) - modified_messages.append(continue_message) - return modified_messages diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 12efb47e067..355c23ae175 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -858,6 +858,50 @@ def test_ensure_alternating_roles_three_consecutive_assistants(): ] +def test_ensure_alternating_roles_does_not_split_tool_call_chain(): + messages = [ + {"role": "user", "content": "Search for X"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "results"}, + {"role": "user", "content": "Thanks, now do Y"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "Search for X"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "results"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "Thanks, now do Y"}, + ] + + def test_alternating_roles_e2e(): from litellm.llms.custom_httpx.http_handler import HTTPHandler import json From 7e4ec1000718f2f35854552eaf0d3aac43f48fa4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:08:15 +0530 Subject: [PATCH 51/83] Update litellm/litellm_core_utils/prompt_templates/common_utils.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/prompt_templates/common_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 739c3119cc0..f5e3ffdb749 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -299,9 +299,7 @@ def _insert_user_continue_message( while i < len(result_messages): curr_message = result_messages[i] inserted_continue_message = False - if ( - curr_message["role"] == "assistant" - and _counts_for_alternation(curr_message) + if _counts_for_alternation(curr_message) and curr_message["role"] == "assistant": ): j = i - 1 while j >= 0: From ae350ed3708b9e90a5a185e9fb5d1073f6b0dc35 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:09:32 +0530 Subject: [PATCH 52/83] Fix greptile comments --- tests/llm_translation/test_prompt_factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 355c23ae175..f146e2811a9 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -776,6 +776,7 @@ def test_ensure_alternating_roles( def test_ensure_alternating_roles_with_tool_calls(): + """Fixes Regression in #18685 """ messages = [ {"role": "user", "content": "What's the weather?"}, { From 35b3ed58a8bdc190c9256d3bbfc706a63623c2e9 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:19:11 +0530 Subject: [PATCH 53/83] Fix greptile review --- litellm/responses/main.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index cec4565166b..a844b61854b 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -630,10 +630,8 @@ def responses( prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) - if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( - litellm_logging_obj.should_run_prompt_management_hooks( - prompt_id=prompt_id, non_default_params=kwargs - ) + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and litellm_logging_obj.should_run_prompt_management_hooks( + prompt_id=prompt_id, non_default_params=kwargs ): client_input: List[AllMessageValues] = ( [{"role": "user", "content": input}] @@ -655,9 +653,9 @@ def responses( ) input = cast(Union[str, ResponseInputParam], merged_input) local_vars["input"] = input - # Apply prompt_template_optional_params (e.g. temperature, instructions) - # by updating kwargs so they flow into local_vars → response_api_optional_params - kwargs.update(merged_optional_params) + local_vars["model"] = model + for k, v in merged_optional_params.items(): + local_vars[k] = v ######################################################### # Update input and tools with provider-specific file IDs if managed files are used From 0941e4036365ad9a57db604d35d98f0356022b54 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:30:42 +0530 Subject: [PATCH 54/83] fix(prompting): address greptile review - fix SyntaxError, restore backward compat, add trailing tool-call test - Remove stray ): on line 303 (P0 SyntaxError) - Restore backward-compatible trailing-assistant behavior (P1) - Add test_ensure_alternating_roles_trailing_tool_call_assistant - Keep role check alongside _counts_for_alternation (P2 is false positive) Made-with: Cursor --- .../prompt_templates/common_utils.py | 10 ++---- tests/llm_translation/test_prompt_factory.py | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index f5e3ffdb749..d7c1cc708b0 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -300,7 +300,6 @@ def _insert_user_continue_message( curr_message = result_messages[i] inserted_continue_message = False if _counts_for_alternation(curr_message) and curr_message["role"] == "assistant": - ): j = i - 1 while j >= 0: previous_message = result_messages[j] @@ -314,12 +313,9 @@ def _insert_user_continue_message( if not inserted_continue_message: i += 1 - # Handle final message - if ( - result_messages[-1]["role"] == "assistant" - and _counts_for_alternation(result_messages[-1]) - and ensure_alternating_roles - ): + # Handle final message — append user_continue after any trailing assistant, + # including ones with tool_calls, to preserve backward compatibility. + if result_messages[-1]["role"] == "assistant" and ensure_alternating_roles: result_messages.append(continue_message) return result_messages diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index f146e2811a9..b02ed3ebea1 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -903,6 +903,40 @@ def test_ensure_alternating_roles_does_not_split_tool_call_chain(): ] +def test_ensure_alternating_roles_trailing_tool_call_assistant(): + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NYC"}', + }, + } + ], + }, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + # Backward compat: trailing assistant (even with tool_calls) gets user_continue + # appended, then assistant_continue bridges the user→user gap. + assert transformed_messages[-1] == {"role": "user", "content": "Please continue."} + assert transformed_messages[0] == {"role": "user", "content": "What's the weather?"} + assert transformed_messages[1]["role"] == "assistant" + assert transformed_messages[1].get("tool_calls") is not None + + def test_alternating_roles_e2e(): from litellm.llms.custom_httpx.http_handler import HTTPHandler import json From b32f5ea379f4f4792d1b892cf6fdc3e3d63f80af Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:37:52 +0530 Subject: [PATCH 55/83] Fix greptile comments --- litellm/responses/main.py | 63 +++++++++++++++-- .../test_responses_prompt_management.py | 67 ++++++++++++++++++- 2 files changed, 124 insertions(+), 6 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index a844b61854b..862973e6104 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -464,6 +464,49 @@ async def aresponses( # Update local_vars with detected provider (fixes #19782) local_vars["custom_llm_provider"] = custom_llm_provider + ######################################################### + # ASYNC PROMPT MANAGEMENT + ######################################################### + litellm_logging_obj = kwargs.get("litellm_logging_obj", None) + prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) + prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) + + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and litellm_logging_obj.should_run_prompt_management_hooks( + prompt_id=prompt_id, non_default_params=kwargs + ): + if isinstance(input, str): + client_input: List[AllMessageValues] = [ + {"role": "user", "content": input} + ] + else: + client_input = [ + item # type: ignore[misc] + for item in input + if isinstance(item, dict) and "role" in item + ] + ( + model, + merged_input, + merged_optional_params, + ) = await litellm_logging_obj.async_get_chat_completion_prompt( + model=model, + messages=client_input, + non_default_params=kwargs, + prompt_id=prompt_id, + prompt_variables=prompt_variables, + prompt_label=kwargs.get("prompt_label", None), + prompt_version=kwargs.get("prompt_version", None), + ) + input = cast(Union[str, ResponseInputParam], merged_input) + if "/" in model: + _, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model + ) + local_vars["custom_llm_provider"] = custom_llm_provider + for k, v in merged_optional_params.items(): + if k in local_vars: + local_vars[k] = v + func = partial( responses, input=input, @@ -633,11 +676,16 @@ def responses( if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and litellm_logging_obj.should_run_prompt_management_hooks( prompt_id=prompt_id, non_default_params=kwargs ): - client_input: List[AllMessageValues] = ( - [{"role": "user", "content": input}] - if isinstance(input, str) - else cast(List[AllMessageValues], list(input)) - ) + if isinstance(input, str): + client_input: List[AllMessageValues] = [ + {"role": "user", "content": input} + ] + else: + client_input = [ + item # type: ignore[misc] + for item in input + if isinstance(item, dict) and "role" in item + ] ( model, merged_input, @@ -654,6 +702,11 @@ def responses( input = cast(Union[str, ResponseInputParam], merged_input) local_vars["input"] = input local_vars["model"] = model + if "/" in model: + _, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model + ) + local_vars["custom_llm_provider"] = custom_llm_provider for k, v in merged_optional_params.items(): local_vars[k] = v diff --git a/tests/test_litellm/responses/test_responses_prompt_management.py b/tests/test_litellm/responses/test_responses_prompt_management.py index 788fd195341..666555792f3 100644 --- a/tests/test_litellm/responses/test_responses_prompt_management.py +++ b/tests/test_litellm/responses/test_responses_prompt_management.py @@ -158,7 +158,7 @@ class TestResponsesAPIPromptManagement: ] # Simulate get_chat_completion_prompt returning merged optional params # that include a template-defined temperature - merged_kwargs = {"temperature": 0.2, "prompt_id": "t", "litellm_logging_obj": None} + merged_kwargs = {"temperature": 0.2} logging_obj = MagicMock() logging_obj.__class__ = LiteLLMLoggingObj @@ -209,3 +209,68 @@ class TestResponsesAPIPromptManagement: # The model passed to the downstream handler should be the overridden one handler_call_kwargs = mock_handler.call_args.kwargs assert handler_call_kwargs.get("model") == "openai/gpt-4o-mini" + + def test_non_message_input_items_filtered(self): + """[F] Non-message items in ResponseInputParam (e.g. function_call_output) are + filtered out before being passed to the prompt hook, avoiding malformed merges.""" + template_messages: List[AllMessageValues] = [ + {"role": "system", "content": "You are helpful."}, # type: ignore[list-item] + ] + mixed_input = [ + {"role": "user", "content": "Hello"}, + {"type": "function_call_output", "call_id": "abc", "output": "42"}, + ] + logging_obj = _make_logging_obj( + merged_model="openai/gpt-4o", + merged_messages=template_messages + [{"role": "user", "content": "Hello"}], # type: ignore[operator] + ) + + patches = _patch_responses_dispatch() + with patches[0], patches[1], patches[2], patches[3]: + import litellm + litellm.responses( + input=mixed_input, # type: ignore[arg-type] + model="gpt-4o", + prompt_id="filter-test", + litellm_logging_obj=logging_obj, + ) + + call_kwargs = logging_obj.get_chat_completion_prompt.call_args.kwargs + passed_messages = call_kwargs["messages"] + assert all(isinstance(m, dict) and "role" in m for m in passed_messages) + assert len(passed_messages) == 1 + + def test_model_override_re_resolves_provider(self): + """[G] When the prompt template overrides the model to a different provider, + custom_llm_provider is re-resolved so downstream routing uses the correct provider.""" + template_messages: List[AllMessageValues] = [ + {"role": "user", "content": "Hi"}, # type: ignore[list-item] + ] + logging_obj = _make_logging_obj( + merged_model="anthropic/claude-3-5-sonnet", + merged_messages=template_messages, + ) + + patches = _patch_responses_dispatch() + with ( + patch( + "litellm.responses.main.litellm.get_llm_provider", + side_effect=[ + ("gpt-4o", "openai", None, None), + ("claude-3-5-sonnet", "anthropic", None, None), + ], + ), + patches[1], + patches[2], + patches[3] as mock_handler, + ): + import litellm + litellm.responses( + input="Hi", + model="gpt-4o", + prompt_id="cross-provider", + litellm_logging_obj=logging_obj, + ) + + handler_call_kwargs = mock_handler.call_args.kwargs + assert handler_call_kwargs.get("custom_llm_provider") == "anthropic" From 67f5ce9c7c23d476eeb371885b401c625ca61a40 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:41:26 +0530 Subject: [PATCH 56/83] address greptile review feedback (greploop iteration 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Restore backward compat for leading assistant(tool_calls) — always prepend user_continue - Replace partial assertions with full list assertion in trailing tool-call test Made-with: Cursor --- .../prompt_templates/common_utils.py | 8 +++--- tests/llm_translation/test_prompt_factory.py | 25 ++++++++++++++----- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index d7c1cc708b0..8713d0283ed 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -287,11 +287,9 @@ def _insert_user_continue_message( result_messages = messages.copy() # Don't modify the input list continue_message = user_continue_message or DEFAULT_USER_CONTINUE_MESSAGE - # Handle first counted message if it's an assistant message - if ( - result_messages[0]["role"] == "assistant" - and _counts_for_alternation(result_messages[0]) - ): + # Handle first message if it's an assistant message — always prepend + # user_continue regardless of tool_calls, to preserve backward compatibility. + if result_messages[0]["role"] == "assistant": result_messages.insert(0, continue_message) # Handle consecutive assistant messages in the counted sequence diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index b02ed3ebea1..3a9f267e6a0 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -929,12 +929,25 @@ def test_ensure_alternating_roles_trailing_tool_call_assistant(): ensure_alternating_roles=True, ) - # Backward compat: trailing assistant (even with tool_calls) gets user_continue - # appended, then assistant_continue bridges the user→user gap. - assert transformed_messages[-1] == {"role": "user", "content": "Please continue."} - assert transformed_messages[0] == {"role": "user", "content": "What's the weather?"} - assert transformed_messages[1]["role"] == "assistant" - assert transformed_messages[1].get("tool_calls") is not None + assert transformed_messages == [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NYC"}', + }, + } + ], + }, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "Please continue."}, + ] def test_alternating_roles_e2e(): From d333dc4077b98012b08709ce0ed31881ec06879e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:49:07 +0530 Subject: [PATCH 57/83] address greptile review feedback (greploop iteration 1) - Fix async path: call async_get_chat_completion_prompt in aresponses() before executor dispatch, mirroring acompletion() in main.py. Discard merged_optional_params in async path (sync responses() handles them via local_vars), avoiding TypeError from duplicate kwargs in partial(). - Fix provider re-resolution: replace "/" in model heuristic with model != original_model comparison so bare model names are handled. - Add 3 async tests covering hook invocation, optional param propagation, and non-message item filtering in aresponses(). Made-with: Cursor --- litellm/responses/main.py | 12 +- .../test_responses_prompt_management.py | 119 ++++++++++++++++-- 2 files changed, 117 insertions(+), 14 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 862973e6104..0e568363558 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -470,6 +470,7 @@ async def aresponses( litellm_logging_obj = kwargs.get("litellm_logging_obj", None) prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) + original_model = model if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and litellm_logging_obj.should_run_prompt_management_hooks( prompt_id=prompt_id, non_default_params=kwargs @@ -487,7 +488,7 @@ async def aresponses( ( model, merged_input, - merged_optional_params, + _, ) = await litellm_logging_obj.async_get_chat_completion_prompt( model=model, messages=client_input, @@ -498,14 +499,10 @@ async def aresponses( prompt_version=kwargs.get("prompt_version", None), ) input = cast(Union[str, ResponseInputParam], merged_input) - if "/" in model: + if model != original_model: _, custom_llm_provider, _, _ = litellm.get_llm_provider( model=model ) - local_vars["custom_llm_provider"] = custom_llm_provider - for k, v in merged_optional_params.items(): - if k in local_vars: - local_vars[k] = v func = partial( responses, @@ -672,6 +669,7 @@ def responses( ######################################################### prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) + original_model = model if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and litellm_logging_obj.should_run_prompt_management_hooks( prompt_id=prompt_id, non_default_params=kwargs @@ -702,7 +700,7 @@ def responses( input = cast(Union[str, ResponseInputParam], merged_input) local_vars["input"] = input local_vars["model"] = model - if "/" in model: + if model != original_model: _, custom_llm_provider, _, _ = litellm.get_llm_provider( model=model ) diff --git a/tests/test_litellm/responses/test_responses_prompt_management.py b/tests/test_litellm/responses/test_responses_prompt_management.py index 666555792f3..9defaceed8e 100644 --- a/tests/test_litellm/responses/test_responses_prompt_management.py +++ b/tests/test_litellm/responses/test_responses_prompt_management.py @@ -6,10 +6,18 @@ Covers: B) list input is merged with the template C) no prompt_id → hook is skipped, input is unchanged D) model override from the prompt template is applied + E) prompt_template_optional_params flow into the request + F) non-message items in input are filtered out + G) model override re-resolves provider + H) async path calls async_get_chat_completion_prompt + I) async path propagates optional params to downstream handler """ +import asyncio from typing import List -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.llms.openai import AllMessageValues @@ -22,18 +30,19 @@ def _make_logging_obj( merged_model: str, merged_messages: List[AllMessageValues], should_run: bool = True, + merged_optional_params: dict = None, ) -> MagicMock: """Return a mock LiteLLMLoggingObj pre-configured for prompt management.""" + if merged_optional_params is None: + merged_optional_params = {} logging_obj = MagicMock() - # Make isinstance(logging_obj, LiteLLMLoggingObj) return True logging_obj.__class__ = LiteLLMLoggingObj logging_obj.should_run_prompt_management_hooks.return_value = should_run - logging_obj.get_chat_completion_prompt.return_value = ( - merged_model, - merged_messages, - {}, + prompt_return = (merged_model, merged_messages, merged_optional_params) + logging_obj.get_chat_completion_prompt.return_value = prompt_return + logging_obj.async_get_chat_completion_prompt = AsyncMock( + return_value=prompt_return ) - # Instance attribute accessed by post-call metadata utilities logging_obj.model_call_details = {} return logging_obj @@ -274,3 +283,99 @@ class TestResponsesAPIPromptManagement: handler_call_kwargs = mock_handler.call_args.kwargs assert handler_call_kwargs.get("custom_llm_provider") == "anthropic" + + +class TestAsyncResponsesAPIPromptManagement: + """Tests for the async aresponses() prompt management path. + + aresponses() calls async_get_chat_completion_prompt at the outer async level + (for async-only prompt loggers), then delegates to responses() via + run_in_executor where the sync hook also runs — mirroring acompletion() in + main.py. Optional params are handled by the sync responses() path. + """ + + @pytest.mark.asyncio + async def test_async_calls_async_hook(self): + """[H] aresponses() invokes async_get_chat_completion_prompt before + dispatching to the sync responses() path.""" + template_messages: List[AllMessageValues] = [ + {"role": "system", "content": "You are helpful."}, # type: ignore[list-item] + ] + logging_obj = _make_logging_obj( + merged_model="openai/gpt-4o", + merged_messages=template_messages + [{"role": "user", "content": "Hi"}], # type: ignore[list-item] + ) + + patches = _patch_responses_dispatch() + with patches[0], patches[1], patches[2], patches[3]: + import litellm + await litellm.aresponses( + input="Hi", + model="gpt-4o", + prompt_id="async-test", + prompt_variables={}, + litellm_logging_obj=logging_obj, + ) + + logging_obj.async_get_chat_completion_prompt.assert_called_once() + call_kwargs = logging_obj.async_get_chat_completion_prompt.call_args.kwargs + assert call_kwargs["prompt_id"] == "async-test" + + @pytest.mark.asyncio + async def test_async_optional_params_propagated(self): + """[I] Template-defined optional params (e.g. temperature) reach the downstream + handler when called via aresponses(). The sync responses() path applies them + via local_vars.""" + template_messages: List[AllMessageValues] = [ + {"role": "user", "content": "Hello"}, # type: ignore[list-item] + ] + logging_obj = _make_logging_obj( + merged_model="openai/gpt-4o", + merged_messages=template_messages, + merged_optional_params={"temperature": 0.7}, + ) + + patches = _patch_responses_dispatch() + with patches[0], patches[1], patches[2], patches[3] as mock_handler: + import litellm + await litellm.aresponses( + input="Hello", + model="gpt-4o", + prompt_id="async-temp", + litellm_logging_obj=logging_obj, + ) + + handler_call_kwargs = mock_handler.call_args.kwargs + request_params = handler_call_kwargs.get("responses_api_request", {}) + assert request_params.get("temperature") == 0.7 + + @pytest.mark.asyncio + async def test_async_non_message_items_filtered(self): + """[J] Non-message items are filtered in the async path too.""" + template_messages: List[AllMessageValues] = [ + {"role": "system", "content": "Be helpful."}, # type: ignore[list-item] + ] + mixed_input = [ + {"role": "user", "content": "Hello"}, + {"type": "function_call_output", "call_id": "abc", "output": "42"}, + ] + logging_obj = _make_logging_obj( + merged_model="openai/gpt-4o", + merged_messages=template_messages + [{"role": "user", "content": "Hello"}], # type: ignore[operator] + ) + + patches = _patch_responses_dispatch() + with patches[0], patches[1], patches[2], patches[3]: + import litellm + await litellm.aresponses( + input=mixed_input, # type: ignore[arg-type] + model="gpt-4o", + prompt_id="async-filter", + litellm_logging_obj=logging_obj, + ) + + logging_obj.async_get_chat_completion_prompt.assert_called_once() + call_kwargs = logging_obj.async_get_chat_completion_prompt.call_args.kwargs + passed_messages = call_kwargs["messages"] + assert all(isinstance(m, dict) and "role" in m for m in passed_messages) + assert len(passed_messages) == 1 From 22fc08d602598f5b5cbe2293ccbd146bb748e622 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:59:16 +0530 Subject: [PATCH 58/83] fix(prompting): revert _insert_assistant_continue_message to adjacent-check logic Restore backward-compatible behavior: only insert assistant_continue between directly adjacent user messages, not across tool-call chains. The _counts_for_alternation skip logic was a silent behavioral change for [user, assistant(tc), tool, user] sequences. Made-with: Cursor --- .../prompt_templates/common_utils.py | 42 +++++-------------- tests/llm_translation/test_prompt_factory.py | 3 +- 2 files changed, 12 insertions(+), 33 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 8713d0283ed..eb3755b71fe 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -326,45 +326,25 @@ def _insert_assistant_continue_message( ) -> List[AllMessageValues]: """ Add assistant continuation messages between consecutive user messages. - Skips tool messages and assistant messages with tool calls in the - alternation check, matching strict templates like llama.cpp. - Args: - messages: List of message dictionaries - assistant_continue_message: Optional custom assistant message - ensure_alternating_roles: Whether to enforce alternating roles - - Returns: - Modified list of messages with inserted assistant messages + Only checks directly adjacent messages to preserve backward compatibility. """ if not ensure_alternating_roles or len(messages) <= 1: return messages + continue_message = assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE - insert_before_indexes = set() - - for i, message in enumerate(messages): - if message.get("role") != "user": - continue - - next_counted_index = i + 1 - while next_counted_index < len(messages) and not _counts_for_alternation( - messages[next_counted_index] - ): - next_counted_index += 1 - - if ( - next_counted_index < len(messages) - and messages[next_counted_index].get("role") == "user" - ): - # Insert before the next counted user turn. - # This avoids splitting assistant tool-call -> tool chains. - insert_before_indexes.add(next_counted_index) modified_messages: List[AllMessageValues] = [] - for idx, message in enumerate(messages): - if idx in insert_before_indexes: + for i, message in enumerate(messages): + if ( + i < len(messages) - 1 + and message.get("role") == "user" + and messages[i + 1].get("role") == "user" + ): + modified_messages.append(message) modified_messages.append(continue_message) - modified_messages.append(message) + else: + modified_messages.append(message) return modified_messages diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 3a9f267e6a0..fe46c24a298 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -860,6 +860,7 @@ def test_ensure_alternating_roles_three_consecutive_assistants(): def test_ensure_alternating_roles_does_not_split_tool_call_chain(): + """Tool-call chains [user, assistant(tc), tool, user] are preserved as-is.""" messages = [ {"role": "user", "content": "Search for X"}, { @@ -898,7 +899,6 @@ def test_ensure_alternating_roles_does_not_split_tool_call_chain(): ], }, {"role": "tool", "tool_call_id": "c1", "content": "results"}, - {"role": "assistant", "content": "Please continue."}, {"role": "user", "content": "Thanks, now do Y"}, ] @@ -945,7 +945,6 @@ def test_ensure_alternating_roles_trailing_tool_call_assistant(): } ], }, - {"role": "assistant", "content": "Please continue."}, {"role": "user", "content": "Please continue."}, ] From 021540b2e2d7083cecfadfdcc4cd1f08abb43e31 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 17:09:28 +0530 Subject: [PATCH 59/83] fix: prevent double prompt management in async path, preserve optional params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - aresponses() now pops prompt_id from kwargs after the async hook runs and passes merged_optional_params via _async_prompt_merged_params. responses() checks for this internal kwarg first and skips the sync hook entirely when present — eliminating double-merge of template messages. - merged_optional_params from async_get_chat_completion_prompt is no longer discarded (_); it flows through to local_vars in responses(). - Async tests now assert get_chat_completion_prompt.assert_not_called() to directly detect any double-execution regression. Made-with: Cursor --- litellm/responses/main.py | 97 +++++++++++-------- .../test_responses_prompt_management.py | 22 +++-- 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 0e568363558..af2976cd54d 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -466,6 +466,11 @@ async def aresponses( ######################################################### # ASYNC PROMPT MANAGEMENT + # Run the async hook here so async-only prompt loggers are honoured. + # Then pop prompt_id from kwargs so the sync responses() path does NOT + # re-run the hook (which would double-prepend template messages). + # Pass merged_optional_params via an internal kwarg so responses() + # can apply them to local_vars without re-invoking the hook. ######################################################### litellm_logging_obj = kwargs.get("litellm_logging_obj", None) prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) @@ -488,7 +493,7 @@ async def aresponses( ( model, merged_input, - _, + merged_optional_params, ) = await litellm_logging_obj.async_get_chat_completion_prompt( model=model, messages=client_input, @@ -503,6 +508,8 @@ async def aresponses( _, custom_llm_provider, _, _ = litellm.get_llm_provider( model=model ) + kwargs.pop("prompt_id", None) + kwargs["_async_prompt_merged_params"] = merged_optional_params func = partial( responses, @@ -666,47 +673,57 @@ def responses( ######################################################### # PROMPT MANAGEMENT + # If aresponses() already ran the async hook, it pops prompt_id and + # passes the result via _async_prompt_merged_params — apply those + # directly and skip the sync hook to avoid double-merging. ######################################################### - prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) - prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) - original_model = model - - if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and litellm_logging_obj.should_run_prompt_management_hooks( - prompt_id=prompt_id, non_default_params=kwargs - ): - if isinstance(input, str): - client_input: List[AllMessageValues] = [ - {"role": "user", "content": input} - ] - else: - client_input = [ - item # type: ignore[misc] - for item in input - if isinstance(item, dict) and "role" in item - ] - ( - model, - merged_input, - merged_optional_params, - ) = litellm_logging_obj.get_chat_completion_prompt( - model=model, - messages=client_input, - non_default_params=kwargs, - prompt_id=prompt_id, - prompt_variables=prompt_variables, - prompt_label=kwargs.get("prompt_label", None), - prompt_version=kwargs.get("prompt_version", None), - ) - input = cast(Union[str, ResponseInputParam], merged_input) - local_vars["input"] = input - local_vars["model"] = model - if model != original_model: - _, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model - ) - local_vars["custom_llm_provider"] = custom_llm_provider - for k, v in merged_optional_params.items(): + _async_merged = kwargs.pop("_async_prompt_merged_params", None) + if _async_merged is not None: + for k, v in _async_merged.items(): local_vars[k] = v + else: + prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) + prompt_variables = cast( + Optional[dict], kwargs.get("prompt_variables", None) + ) + original_model = model + + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and litellm_logging_obj.should_run_prompt_management_hooks( + prompt_id=prompt_id, non_default_params=kwargs + ): + if isinstance(input, str): + client_input: List[AllMessageValues] = [ + {"role": "user", "content": input} + ] + else: + client_input = [ + item # type: ignore[misc] + for item in input + if isinstance(item, dict) and "role" in item + ] + ( + model, + merged_input, + merged_optional_params, + ) = litellm_logging_obj.get_chat_completion_prompt( + model=model, + messages=client_input, + non_default_params=kwargs, + prompt_id=prompt_id, + prompt_variables=prompt_variables, + prompt_label=kwargs.get("prompt_label", None), + prompt_version=kwargs.get("prompt_version", None), + ) + input = cast(Union[str, ResponseInputParam], merged_input) + local_vars["input"] = input + local_vars["model"] = model + if model != original_model: + _, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model + ) + local_vars["custom_llm_provider"] = custom_llm_provider + for k, v in merged_optional_params.items(): + local_vars[k] = v ######################################################### # Update input and tools with provider-specific file IDs if managed files are used diff --git a/tests/test_litellm/responses/test_responses_prompt_management.py b/tests/test_litellm/responses/test_responses_prompt_management.py index 9defaceed8e..f49679fc400 100644 --- a/tests/test_litellm/responses/test_responses_prompt_management.py +++ b/tests/test_litellm/responses/test_responses_prompt_management.py @@ -288,16 +288,16 @@ class TestResponsesAPIPromptManagement: class TestAsyncResponsesAPIPromptManagement: """Tests for the async aresponses() prompt management path. - aresponses() calls async_get_chat_completion_prompt at the outer async level - (for async-only prompt loggers), then delegates to responses() via - run_in_executor where the sync hook also runs — mirroring acompletion() in - main.py. Optional params are handled by the sync responses() path. + aresponses() calls async_get_chat_completion_prompt at the outer async + level, then pops prompt_id from kwargs and passes merged_optional_params + via an internal kwarg. The sync responses() path sees no prompt_id and + skips the sync hook entirely — preventing double-merge of template messages. """ @pytest.mark.asyncio - async def test_async_calls_async_hook(self): - """[H] aresponses() invokes async_get_chat_completion_prompt before - dispatching to the sync responses() path.""" + async def test_async_calls_async_hook_not_sync(self): + """[H] aresponses() invokes async_get_chat_completion_prompt and the + sync get_chat_completion_prompt is NOT called (no double-merge).""" template_messages: List[AllMessageValues] = [ {"role": "system", "content": "You are helpful."}, # type: ignore[list-item] ] @@ -318,14 +318,14 @@ class TestAsyncResponsesAPIPromptManagement: ) logging_obj.async_get_chat_completion_prompt.assert_called_once() + logging_obj.get_chat_completion_prompt.assert_not_called() call_kwargs = logging_obj.async_get_chat_completion_prompt.call_args.kwargs assert call_kwargs["prompt_id"] == "async-test" @pytest.mark.asyncio async def test_async_optional_params_propagated(self): - """[I] Template-defined optional params (e.g. temperature) reach the downstream - handler when called via aresponses(). The sync responses() path applies them - via local_vars.""" + """[I] Template-defined optional params (e.g. temperature) from the async + hook reach the downstream handler — they are NOT silently discarded.""" template_messages: List[AllMessageValues] = [ {"role": "user", "content": "Hello"}, # type: ignore[list-item] ] @@ -345,6 +345,7 @@ class TestAsyncResponsesAPIPromptManagement: litellm_logging_obj=logging_obj, ) + logging_obj.get_chat_completion_prompt.assert_not_called() handler_call_kwargs = mock_handler.call_args.kwargs request_params = handler_call_kwargs.get("responses_api_request", {}) assert request_params.get("temperature") == 0.7 @@ -375,6 +376,7 @@ class TestAsyncResponsesAPIPromptManagement: ) logging_obj.async_get_chat_completion_prompt.assert_called_once() + logging_obj.get_chat_completion_prompt.assert_not_called() call_kwargs = logging_obj.async_get_chat_completion_prompt.call_args.kwargs passed_messages = call_kwargs["messages"] assert all(isinstance(m, dict) and "role" in m for m in passed_messages) From f29b4981a0fca42eb7c3918da1649a9036d74e49 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 17:13:42 +0530 Subject: [PATCH 60/83] fix(prompting): preserve separator for assistant(tc)->assistant edge case When scanning backward over counted messages, preserve old behavior for adjacent assistant turns by inserting user_continue if the immediate previous raw message is assistant. This handles malformed assistant(tool_calls)->assistant(no-tool-calls) inputs without splitting valid assistant(tool_calls)->tool chains. Made-with: Cursor --- .../prompt_templates/common_utils.py | 27 +++++++---- tests/llm_translation/test_prompt_factory.py | 48 +++++++++++++++++++ 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index eb3755b71fe..2efd90e0c2f 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -298,16 +298,23 @@ def _insert_user_continue_message( curr_message = result_messages[i] inserted_continue_message = False if _counts_for_alternation(curr_message) and curr_message["role"] == "assistant": - j = i - 1 - while j >= 0: - previous_message = result_messages[j] - if _counts_for_alternation(previous_message): - if previous_message["role"] == "assistant": - result_messages.insert(i, continue_message) - i += 2 - inserted_continue_message = True - break - j -= 1 + # Preserve old behavior for malformed adjacent assistant sequences like + # assistant(tool_calls) -> assistant(no-tool-calls) with no tool message. + if i > 0 and result_messages[i - 1].get("role") == "assistant": + result_messages.insert(i, continue_message) + i += 2 + inserted_continue_message = True + else: + j = i - 1 + while j >= 0: + previous_message = result_messages[j] + if _counts_for_alternation(previous_message): + if previous_message["role"] == "assistant": + result_messages.insert(i, continue_message) + i += 2 + inserted_continue_message = True + break + j -= 1 if not inserted_continue_message: i += 1 diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index fe46c24a298..64556c3f26d 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -903,6 +903,54 @@ def test_ensure_alternating_roles_does_not_split_tool_call_chain(): ] +def test_ensure_alternating_roles_assistant_tool_call_then_assistant(): + """ + Preserve old behavior for malformed adjacent assistant turns: + [assistant(tool_calls), assistant(no-tool-calls), user] should insert + user_continue between assistant messages. + """ + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + } + ], + }, + {"role": "assistant", "content": "Here's what I found."}, + {"role": "user", "content": "Thanks"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "Please continue."}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + } + ], + }, + {"role": "user", "content": "Please continue."}, + {"role": "assistant", "content": "Here's what I found."}, + {"role": "user", "content": "Thanks"}, + ] + + def test_ensure_alternating_roles_trailing_tool_call_assistant(): messages = [ {"role": "user", "content": "What's the weather?"}, From 17efd96e6173bd2eee3ee5cc8ee1640dc0799cb4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 17:49:03 +0530 Subject: [PATCH 61/83] docs(vertex): add concise PayGo/Priority guide with cost-tracking flow Document how to send Vertex Priority PayGo headers and explain how trafficType maps to service-tier pricing in LiteLLM, including an embedded flow diagram for quick understanding. Made-with: Cursor --- .../docs/tutorials/vertex_ai_pay_go.md | 151 ++++++++++++++++++ .../static/img/vertex_cost_tracking_flow.svg | 62 +++++++ 2 files changed, 213 insertions(+) create mode 100644 docs/my-website/docs/tutorials/vertex_ai_pay_go.md create mode 100644 docs/my-website/static/img/vertex_cost_tracking_flow.svg diff --git a/docs/my-website/docs/tutorials/vertex_ai_pay_go.md b/docs/my-website/docs/tutorials/vertex_ai_pay_go.md new file mode 100644 index 00000000000..625aff35e1b --- /dev/null +++ b/docs/my-website/docs/tutorials/vertex_ai_pay_go.md @@ -0,0 +1,151 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Vertex AI PayGo and Priority + +## Priority PayGo + +LiteLLM supports Priority PayGo. +Send a priority header, get priority queueing, and pay priority token rates. + +:::info Which models support Priority PayGo? +As of this writing: `gemini/gemini-2.5-pro`, `vertex_ai/gemini-3-pro-preview`, `vertex_ai/gemini-3.1-pro-preview`, `vertex_ai/gemini-3-flash-preview`, and their variants. +Check `supports_service_tier: true` in LiteLLM's [model pricing JSON](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). +::: + +### Send a priority request + +Use this header: + +`X-Vertex-AI-LLM-Shared-Request-Type: priority` + + + + +```python +import litellm + +response = litellm.completion( + model="vertex_ai/gemini-3-pro-preview", + messages=[{"role": "user", "content": "Summarize the Gettysburg Address."}], + vertex_project="YOUR_PROJECT_ID", + vertex_location="us-central1", + extra_headers={"X-Vertex-AI-LLM-Shared-Request-Type": "priority"}, +) + +print(response.choices[0].message.content) +``` + + + + +```yaml title="config.yaml" +model_list: + - model_name: gemini-priority + litellm_params: + model: vertex_ai/gemini-3-pro-preview + vertex_project: "YOUR_PROJECT_ID" + vertex_location: "us-central1" + vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS + extra_headers: + X-Vertex-AI-LLM-Shared-Request-Type: priority +``` + +```bash +curl http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer sk-your-key" \ + -H "Content-Type: application/json" \ + -d '{"model": "gemini-priority", "messages": [{"role": "user", "content": "Hello"}]}' +``` + + + + +Use `x-pass-` so LiteLLM forwards provider-specific headers. + +```bash +MODEL_ID="gemini-3-pro-preview-0325" +PROJECT_ID="YOUR_PROJECT_ID" + +curl -X POST \ + "${LITELLM_PROXY_BASE_URL}/vertex_ai/v1/projects/${PROJECT_ID}/locations/global/publishers/google/models/${MODEL_ID}:generateContent" \ + -H "Authorization: Bearer sk-your-litellm-key" \ + -H "Content-Type: application/json" \ + -H "x-pass-X-Vertex-AI-LLM-Shared-Request-Type: priority" \ + -d '{"contents": [{"role": "user", "parts": [{"text": "Hello!"}]}]}' +``` + + + + +### How cost tracking works + +![Vertex AI Priority PayGo Cost Tracking Flow](/img/vertex_cost_tracking_flow.svg) + +**`trafficType` → `service_tier` mapping** + +| `usageMetadata.trafficType` | `service_tier` | Pricing keys used | +|---|---|---| +| `ON_DEMAND` | `None` | `input_cost_per_token` | +| `ON_DEMAND_PRIORITY` | `"priority"` | `input_cost_per_token_priority` | +| `FLEX` / `BATCH` | `"flex"` | `input_cost_per_token_flex` | + +If a tier-specific key is missing, LiteLLM falls back to standard pricing keys. + +--- + +## Standard PayGo vs Provisioned Throughput + +This is a different header from priority routing: + +| Header value | Behavior | +|---|---| +| `X-Vertex-AI-LLM-Request-Type: shared` | Force standard PayGo (bypass PT) | +| `X-Vertex-AI-LLM-Request-Type: dedicated` | Force Provisioned Throughput only (`429` if exhausted) | + +### Native route example + +```python +import litellm + +response = litellm.completion( + model="vertex_ai/gemini-2.0-flash", + messages=[{"role": "user", "content": "Hello!"}], + vertex_project="YOUR_PROJECT_ID", + vertex_location="us-central1", + extra_headers={"X-Vertex-AI-LLM-Request-Type": "shared"}, +) +``` + +### Pass-through example + +```bash +MODEL_ID="gemini-2.0-flash-001" +PROJECT_ID="YOUR_PROJECT_ID" + +curl -X POST \ + "${LITELLM_PROXY_BASE_URL}/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent" \ + -H "Authorization: Bearer sk-your-litellm-key" \ + -H "Content-Type: application/json" \ + -H "x-pass-X-Vertex-AI-LLM-Request-Type: shared" \ + -d '{ + "contents": [{"role": "user", "parts": [{"text": "Hello!"}]}] + }' +``` + +--- + +## Troubleshooting + +**Q: What does `403 Permission denied` or `IAM_PERMISSION_DENIED` mean?** +A: The service account or Application Default Credentials (ADC) user does not have the `roles/aiplatform.user` role. To resolve this, re-run the `gcloud projects add-iam-policy-binding` command as shown above in the guide. + +**Q: What should I do if I get a `429 Quota exceeded` error?** +A: This means you've hit the per-region QPM (queries per minute) or TPM (tokens per minute) quota. You can: +- Request a quota increase from the [GCP Quotas console](https://console.cloud.google.com/iam-admin/quotas) +- Add more regions to your LiteLLM configuration for load balancing (see the region balancing guide above) +- Upgrade to [Provisioned Throughput](https://cloud.google.com/vertex-ai/generative-ai/docs/provisioned-throughput) for guaranteed capacity + +**Q: How do I fix the `VERTEXAI_PROJECT not set` error?** +A: Either pass the `vertex_project` parameter explicitly in your LiteLLM call, or set the `VERTEXAI_PROJECT` environment variable before running your code. + diff --git a/docs/my-website/static/img/vertex_cost_tracking_flow.svg b/docs/my-website/static/img/vertex_cost_tracking_flow.svg new file mode 100644 index 00000000000..d808dd2e363 --- /dev/null +++ b/docs/my-website/static/img/vertex_cost_tracking_flow.svg @@ -0,0 +1,62 @@ + + + + + + + + + + + HTTP request + X-Vertex-AI-LLM-Shared-Request-Type: priority + + + + + Vertex AI + + + + + Vertex response + usageMetadata.trafficType = ON_DEMAND_PRIORITY + + + + + + + + + LiteLLM stores it + _hidden_params.provider_specific_fields.traffic_type + + + + + + + + + completion_cost() + Maps traffic_type service_tier = "priority" + + + + + + + + + Pricing lookup + input/output_cost_per_token_priority + + + + ` + a + b + c + d + From 8e943929a2a3802f03036437521ce73cdd2a2c50 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 17:52:51 +0530 Subject: [PATCH 62/83] docs(sidebar): add vertex PayGo tutorial under Spend Tracking Made-with: Cursor --- docs/my-website/sidebars.js | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 79a0279bad5..56d1bb8b553 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -528,6 +528,7 @@ const sidebars = { label: "Spend Tracking", items: [ "proxy/cost_tracking", + "tutorials/vertex_ai_pay_go", "proxy/request_tags", "proxy/custom_pricing", "proxy/pricing_calculator", From b56fdf188ed95d7db6457caec8f3008c4f0e97f4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 17:57:27 +0530 Subject: [PATCH 63/83] Fix greptile review --- .../docs/tutorials/vertex_ai_pay_go.md | 4 +-- .../static/img/vertex_cost_tracking_flow.svg | 27 ++++++++++--------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/my-website/docs/tutorials/vertex_ai_pay_go.md b/docs/my-website/docs/tutorials/vertex_ai_pay_go.md index 625aff35e1b..b40a8b2157f 100644 --- a/docs/my-website/docs/tutorials/vertex_ai_pay_go.md +++ b/docs/my-website/docs/tutorials/vertex_ai_pay_go.md @@ -124,7 +124,7 @@ MODEL_ID="gemini-2.0-flash-001" PROJECT_ID="YOUR_PROJECT_ID" curl -X POST \ - "${LITELLM_PROXY_BASE_URL}/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent" \ + "${LITELLM_PROXY_BASE_URL}/vertex_ai/v1/projects/${PROJECT_ID}/locations/global/publishers/google/models/${MODEL_ID}:generateContent" \ -H "Authorization: Bearer sk-your-litellm-key" \ -H "Content-Type: application/json" \ -H "x-pass-X-Vertex-AI-LLM-Request-Type: shared" \ @@ -143,7 +143,7 @@ A: The service account or Application Default Credentials (ADC) user does not ha **Q: What should I do if I get a `429 Quota exceeded` error?** A: This means you've hit the per-region QPM (queries per minute) or TPM (tokens per minute) quota. You can: - Request a quota increase from the [GCP Quotas console](https://console.cloud.google.com/iam-admin/quotas) -- Add more regions to your LiteLLM configuration for load balancing (see the region balancing guide above) +- Add more regions to your LiteLLM configuration for load balancing - Upgrade to [Provisioned Throughput](https://cloud.google.com/vertex-ai/generative-ai/docs/provisioned-throughput) for guaranteed capacity **Q: How do I fix the `VERTEXAI_PROJECT not set` error?** diff --git a/docs/my-website/static/img/vertex_cost_tracking_flow.svg b/docs/my-website/static/img/vertex_cost_tracking_flow.svg index d808dd2e363..d607d072dd5 100644 --- a/docs/my-website/static/img/vertex_cost_tracking_flow.svg +++ b/docs/my-website/static/img/vertex_cost_tracking_flow.svg @@ -6,7 +6,7 @@ - + HTTP request X-Vertex-AI-LLM-Shared-Request-Type: priority @@ -17,7 +17,7 @@ Vertex AI - + Vertex response usageMetadata.trafficType = ON_DEMAND_PRIORITY @@ -27,7 +27,7 @@ - + LiteLLM stores it _hidden_params.provider_specific_fields.traffic_type @@ -37,26 +37,27 @@ - + completion_cost() - Maps traffic_type service_tier = "priority" + Maps traffic_type → service_tier = "priority" - + Pricing lookup input/output_cost_per_token_priority - - ` - a - b - c - d - + + ① + ② + ③ + ④ + ⑤ + + \ No newline at end of file From ea80a19a3970f338b398dafb6f57881f52bad30a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 18:07:25 +0530 Subject: [PATCH 64/83] Fix greptile review --- docs/my-website/docs/tutorials/vertex_ai_pay_go.md | 2 +- .../static/img/vertex_cost_tracking_flow.svg | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/my-website/docs/tutorials/vertex_ai_pay_go.md b/docs/my-website/docs/tutorials/vertex_ai_pay_go.md index b40a8b2157f..87197e5bad5 100644 --- a/docs/my-website/docs/tutorials/vertex_ai_pay_go.md +++ b/docs/my-website/docs/tutorials/vertex_ai_pay_go.md @@ -138,7 +138,7 @@ curl -X POST \ ## Troubleshooting **Q: What does `403 Permission denied` or `IAM_PERMISSION_DENIED` mean?** -A: The service account or Application Default Credentials (ADC) user does not have the `roles/aiplatform.user` role. To resolve this, re-run the `gcloud projects add-iam-policy-binding` command as shown above in the guide. +A: The service account or Application Default Credentials (ADC) user does not have the `roles/aiplatform.user` role. To resolve this, re-run the `gcloud projects add-iam-policy-binding`. **Q: What should I do if I get a `429 Quota exceeded` error?** A: This means you've hit the per-region QPM (queries per minute) or TPM (tokens per minute) quota. You can: diff --git a/docs/my-website/static/img/vertex_cost_tracking_flow.svg b/docs/my-website/static/img/vertex_cost_tracking_flow.svg index d607d072dd5..c3b2e33a073 100644 --- a/docs/my-website/static/img/vertex_cost_tracking_flow.svg +++ b/docs/my-website/static/img/vertex_cost_tracking_flow.svg @@ -6,7 +6,7 @@ - + HTTP request X-Vertex-AI-LLM-Shared-Request-Type: priority @@ -17,7 +17,7 @@ Vertex AI - + Vertex response usageMetadata.trafficType = ON_DEMAND_PRIORITY @@ -27,7 +27,7 @@ - + LiteLLM stores it _hidden_params.provider_specific_fields.traffic_type @@ -37,7 +37,7 @@ - + completion_cost() Maps traffic_type → service_tier = "priority" @@ -47,7 +47,7 @@ - + Pricing lookup input/output_cost_per_token_priority From 3de0a2a834c00f740f02de11db8f8b394efe4f69 Mon Sep 17 00:00:00 2001 From: Milan Date: Wed, 18 Mar 2026 22:59:11 +0000 Subject: [PATCH 65/83] docs: encrypted_content_affinity requires LiteLLM >= 1.82.3 Made-with: Cursor --- docs/my-website/docs/proxy/config_settings.md | 2 +- docs/my-website/docs/proxy/load_balancing.md | 2 +- docs/my-website/docs/response_api.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 1d6fc1b03bd..ae5f5a4d0e7 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -361,7 +361,7 @@ router_settings: | redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** | | cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. | | router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) | -| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity` (**requires LiteLLM >= 1.82.1**), `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` | +| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity` (requires LiteLLM >= 1.82.3), `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` | | deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). | | ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. | | search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) | diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 313df99b259..74b3e8a5117 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -352,7 +352,7 @@ If `order=1` deployment is unavailable (e.g., rate-limited), the router falls ba When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or organizations), encrypted content items (like `rs_...` reasoning items) can only be decrypted by the originating API key. -**Solution:** Use the `encrypted_content_affinity` pre-call check (**requires LiteLLM >= 1.82.1**) to automatically route follow-up requests containing encrypted items to the correct deployment: +**Solution:** Use the `encrypted_content_affinity` pre-call check (requires LiteLLM >= 1.82.3) to automatically route follow-up requests containing encrypted items to the correct deployment: ```yaml model_list: diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index 66aa2e1ad95..84f662812fb 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -1160,12 +1160,12 @@ follow_up = await router.aresponses( To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks` in your proxy config.yaml. - `responses_api_deployment_check`: high priority routing when `previous_response_id` is provided -- `encrypted_content_affinity`: **[Recommended]** content-aware routing for encrypted items (e.g., `rs_...` reasoning items) (**requires LiteLLM >= 1.82.1**) +- `encrypted_content_affinity`: **[Recommended]** content-aware routing for encrypted items (e.g., `rs_...` reasoning items) (**requires LiteLLM >= 1.82.3**) - `session_affinity`: sticky sessions based on session id (takes priority over `deployment_affinity`) - `deployment_affinity`: sticky sessions based on user key (applies even without `previous_response_id`) :::tip Recommended: Use `encrypted_content_affinity` -For Responses API with load balancing across deployments with **different API keys**, use `encrypted_content_affinity` instead of `deployment_affinity`. It only pins requests that contain encrypted content, avoiding quota reduction while preventing `invalid_encrypted_content` errors. +For Responses API with load balancing across deployments with **different API keys**, use `encrypted_content_affinity` instead of `deployment_affinity`. It only pins requests that contain encrypted content, avoiding quota reduction while preventing `invalid_encrypted_content` errors. (Requires LiteLLM >= 1.82.3.) ::: Notes: From 4829de610278774d360868475316ca144b2c0e27 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 09:28:55 +0530 Subject: [PATCH 66/83] fix(proxy): allow non-admin users to access pass-through subpath routes with auth When a pass-through endpoint has both auth=true and include_subpath=true, non-admin users got 401 errors on subpath requests because only the base path was registered in openai_routes. Now the wildcard path is also registered so the auth check recognizes subpath requests as LLM API routes. Also fixes pre-existing pyright error where logging_obj was possibly unbound in the except block. --- .../pass_through_endpoints.py | 7 ++++ .../proxy/auth/test_route_checks.py | 38 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 0aa99685209..cf6ead974fb 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -651,6 +651,7 @@ async def pass_through_request( # noqa: PLR0915 _parsed_body: Optional[dict] = None # kwargs for pass through endpoint, contains metadata, litellm_params, call_type, litellm_call_id, passthrough_logging_payload kwargs: Optional[dict] = None + logging_obj: Optional[Logging] = None ######################################################### try: @@ -2262,6 +2263,12 @@ async def initialize_pass_through_endpoints( # Add wildcard route for sub-paths if endpoint.get("include_subpath", False) is True: + # Register wildcard path in openai_routes so non-admin users + # can access subpath routes when auth is enabled + if _auth is not None and str(_auth).lower() == "true": + _wildcard_path = _path.rstrip("/") + "/*" + if _wildcard_path not in LiteLLMRoutes.openai_routes.value: + LiteLLMRoutes.openai_routes.value.append(_wildcard_path) InitPassThroughEndpointHelpers.add_subpath_route( app=app, path=_path, diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index f20c14aa611..20fb56e2dc7 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -1329,3 +1329,41 @@ def test_non_org_admin_with_organizations_list(): organization_memberships=[membership], ) assert _user_is_org_admin({"organizations": ["org-1"]}, user_obj) is False + + +def test_pass_through_subpath_auth_with_wildcard_in_openai_routes(): + """ + Test that pass-through endpoints with include_subpath=true and auth=true + are accessible to non-admin users via wildcard route matching. + + When auth=true and include_subpath=true, the wildcard path (e.g. /custom-endpoint/*) + should be added to openai_routes so that subpath requests like + /custom-endpoint/v1/infer are recognized as LLM API routes. + + Regression test for: non-admin users getting 401 "Only proxy admin" error + on pass-through subpath requests. + """ + from litellm.proxy._types import LiteLLMRoutes + + base_path = "/v1/ocr/nvidia/community/nemoretriever-ocr-v1" + wildcard_path = base_path + "/*" + + # Simulate what init_pass_through_endpoints does when auth=true + include_subpath=true + original_routes = LiteLLMRoutes.openai_routes.value[:] + try: + LiteLLMRoutes.openai_routes.value.append(base_path) + LiteLLMRoutes.openai_routes.value.append(wildcard_path) + + # Exact path should match + assert RouteChecks.is_llm_api_route(base_path) is True + + # Subpath should match via wildcard + assert RouteChecks.is_llm_api_route(base_path + "/v1/infer") is True + + # Deeper subpath should also match + assert RouteChecks.is_llm_api_route(base_path + "/v1/some/deep/path") is True + + # Unrelated route should not match + assert RouteChecks.is_llm_api_route("/v1/some-other-endpoint") is False + finally: + LiteLLMRoutes.openai_routes.value[:] = original_routes From 97b7358791052b4056200002d17c76b88479e8f8 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 09:44:30 +0530 Subject: [PATCH 67/83] fix(proxy): dedup openai_routes on reload and clean up on endpoint removal - Add dedup guard for base path registration (prevents unbounded list growth on config reload) - Clean up base path and wildcard path from openai_routes when an endpoint is removed via remove_endpoint_routes - Rewrite test to exercise initialize_pass_through_endpoints directly, covering registration, dedup on reload, and cleanup on removal --- .../pass_through_endpoints.py | 16 ++++- .../proxy/auth/test_route_checks.py | 70 +++++++++++++------ 2 files changed, 64 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index cf6ead974fb..3beaa32b313 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2022,13 +2022,24 @@ class InitPassThroughEndpointHelpers: @staticmethod def remove_endpoint_routes(endpoint_id: str): - """Remove all routes for a specific endpoint ID from the registry""" + """Remove all routes for a specific endpoint ID from the registry + and clean up corresponding entries from LiteLLMRoutes.openai_routes.""" keys_to_remove = [ key for key, value in _registered_pass_through_routes.items() if value["endpoint_id"] == endpoint_id ] for key in keys_to_remove: + route_info = _registered_pass_through_routes[key] + path = route_info.get("path") + if isinstance(path, str): + # Remove base path and wildcard path from openai_routes + openai_routes = LiteLLMRoutes.openai_routes.value + if path in openai_routes: + openai_routes.remove(path) + wildcard_path = path.rstrip("/") + "/*" + if wildcard_path in openai_routes: + openai_routes.remove(wildcard_path) del _registered_pass_through_routes[key] verbose_proxy_logger.debug( "Removed pass-through route from registry: %s", key @@ -2224,7 +2235,8 @@ async def initialize_pass_through_endpoints( ) ) _dependencies = [Depends(user_api_key_auth)] - LiteLLMRoutes.openai_routes.value.append(_path) + if _path not in LiteLLMRoutes.openai_routes.value: + LiteLLMRoutes.openai_routes.value.append(_path) if _target is None: continue diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 20fb56e2dc7..b6c4fa73dfb 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -1331,39 +1331,69 @@ def test_non_org_admin_with_organizations_list(): assert _user_is_org_admin({"organizations": ["org-1"]}, user_obj) is False -def test_pass_through_subpath_auth_with_wildcard_in_openai_routes(): +@pytest.mark.asyncio +async def test_initialize_pass_through_registers_wildcard_for_auth_subpath(): """ - Test that pass-through endpoints with include_subpath=true and auth=true - are accessible to non-admin users via wildcard route matching. + Test that initialize_pass_through_endpoints registers both base path and + wildcard path in openai_routes when auth=true and include_subpath=true, + and that subpath requests pass is_llm_api_route. - When auth=true and include_subpath=true, the wildcard path (e.g. /custom-endpoint/*) - should be added to openai_routes so that subpath requests like - /custom-endpoint/v1/infer are recognized as LLM API routes. - - Regression test for: non-admin users getting 401 "Only proxy admin" error - on pass-through subpath requests. + Also verifies: + - Dedup: calling init twice does not duplicate entries + - Cleanup: removing the endpoint cleans up openai_routes """ from litellm.proxy._types import LiteLLMRoutes + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + initialize_pass_through_endpoints, + ) base_path = "/v1/ocr/nvidia/community/nemoretriever-ocr-v1" wildcard_path = base_path + "/*" - # Simulate what init_pass_through_endpoints does when auth=true + include_subpath=true + endpoint_config = { + "path": base_path, + "target": "https://httpbin.org/post", + "include_subpath": True, + "auth": True, + "headers": {"content-type": "application/json"}, + } + original_routes = LiteLLMRoutes.openai_routes.value[:] try: - LiteLLMRoutes.openai_routes.value.append(base_path) - LiteLLMRoutes.openai_routes.value.append(wildcard_path) + with patch( + "litellm.proxy.proxy_server.app", + MagicMock(), + ), patch( + "litellm.proxy.proxy_server.premium_user", + True, + ), patch( + "litellm.proxy.proxy_server.config_passthrough_endpoints", + None, + ): + await initialize_pass_through_endpoints([endpoint_config]) - # Exact path should match - assert RouteChecks.is_llm_api_route(base_path) is True + # Both base and wildcard paths should be registered + assert base_path in LiteLLMRoutes.openai_routes.value + assert wildcard_path in LiteLLMRoutes.openai_routes.value - # Subpath should match via wildcard - assert RouteChecks.is_llm_api_route(base_path + "/v1/infer") is True + # Subpath requests should pass the auth route check + assert RouteChecks.is_llm_api_route(base_path) is True + assert RouteChecks.is_llm_api_route(base_path + "/v1/infer") is True - # Deeper subpath should also match - assert RouteChecks.is_llm_api_route(base_path + "/v1/some/deep/path") is True + # Calling init again should not duplicate entries + await initialize_pass_through_endpoints([endpoint_config]) + assert LiteLLMRoutes.openai_routes.value.count(base_path) == 1 + assert LiteLLMRoutes.openai_routes.value.count(wildcard_path) == 1 - # Unrelated route should not match - assert RouteChecks.is_llm_api_route("/v1/some-other-endpoint") is False + # Removing the endpoint should clean up openai_routes + # remove_endpoint_routes takes endpoint_id (UUID portion of + # the route key "{id}:exact:{path}:{methods}") + registered = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() + endpoint_ids = {k.split(":")[0] for k in registered} + for eid in endpoint_ids: + InitPassThroughEndpointHelpers.remove_endpoint_routes(eid) + assert base_path not in LiteLLMRoutes.openai_routes.value + assert wildcard_path not in LiteLLMRoutes.openai_routes.value finally: LiteLLMRoutes.openai_routes.value[:] = original_routes From ab1744f9fe197208085483e641c1abd27a30d516 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 09:53:22 +0530 Subject: [PATCH 68/83] fix(proxy): scope wildcard cleanup to subpath entries and restore registry in test - Only remove wildcard path from openai_routes when the route entry has type="subpath", avoiding accidental removal when two endpoints share the same base path but differ in include_subpath - Clean up _registered_pass_through_routes in the test finally block to prevent stale entries from polluting subsequent tests on failure --- .../pass_through_endpoints/pass_through_endpoints.py | 8 ++++---- tests/test_litellm/proxy/auth/test_route_checks.py | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 3beaa32b313..0f676a1feb2 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2033,13 +2033,13 @@ class InitPassThroughEndpointHelpers: route_info = _registered_pass_through_routes[key] path = route_info.get("path") if isinstance(path, str): - # Remove base path and wildcard path from openai_routes openai_routes = LiteLLMRoutes.openai_routes.value if path in openai_routes: openai_routes.remove(path) - wildcard_path = path.rstrip("/") + "/*" - if wildcard_path in openai_routes: - openai_routes.remove(wildcard_path) + if route_info.get("type") == "subpath": + wildcard_path = path.rstrip("/") + "/*" + if wildcard_path in openai_routes: + openai_routes.remove(wildcard_path) del _registered_pass_through_routes[key] verbose_proxy_logger.debug( "Removed pass-through route from registry: %s", key diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index b6c4fa73dfb..83703cd4edd 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -1397,3 +1397,10 @@ async def test_initialize_pass_through_registers_wildcard_for_auth_subpath(): assert wildcard_path not in LiteLLMRoutes.openai_routes.value finally: LiteLLMRoutes.openai_routes.value[:] = original_routes + # Clean up any routes registered during this test to avoid + # polluting the module-level _registered_pass_through_routes + registered = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() + for k in registered: + InitPassThroughEndpointHelpers.remove_endpoint_routes( + k.split(":")[0] + ) From 4dc645fc334a2bce75e345433990ebaea42b1570 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 13:59:59 +0530 Subject: [PATCH 69/83] feat(polling): check rate limits before creating polling ID Move pre-call checks (rate limits, guardrails, budget) to run BEFORE polling ID creation in the background streaming flow. This prevents the edge case where a rate-limited request receives a polling ID that immediately fails. Changes: - Add skip_pre_call_logic parameter to base_process_llm_request to allow skipping pre-call checks (avoiding double-counting of RPM/parallel requests) - Run common_processing_pre_call_logic before generating polling ID in the responses API endpoint. If rate limits/guardrails fail, return error immediately without creating a polling ID - Background streaming task passes skip_pre_call_logic=True to avoid re-running pre-call checks that were already done before polling ID creation - Add tests verifying skip_pre_call_logic parameter works correctly Fixes the edge case where polling_via_cache would return a polling ID for a request that immediately fails due to rate limiting. --- litellm/proxy/common_request_processing.py | 36 +++--- .../proxy/response_api_endpoints/endpoints.py | 32 +++++- .../response_polling/background_streaming.py | 5 +- .../test_response_polling_pre_call_checks.py | 104 ++++++++++++++++++ 4 files changed, 159 insertions(+), 18 deletions(-) create mode 100644 tests/proxy_unit_tests/test_response_polling_pre_call_checks.py diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 72765aab7da..84f9730a370 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -900,6 +900,7 @@ class ProxyBaseLLMRequestProcessing: version: Optional[str] = None, is_streaming_request: Optional[bool] = False, contents: Optional[list] = None, # Add contents parameter + skip_pre_call_logic: bool = False, ) -> Any: """ Common request processing logic for both chat completions and responses API endpoints @@ -909,22 +910,25 @@ class ProxyBaseLLMRequestProcessing: ) self._debug_log_request_payload() - self.data, logging_obj = await self.common_processing_pre_call_logic( - request=request, - general_settings=general_settings, - proxy_logging_obj=proxy_logging_obj, - user_api_key_dict=user_api_key_dict, - version=version, - proxy_config=proxy_config, - user_model=user_model, - user_temperature=user_temperature, - user_request_timeout=user_request_timeout, - user_max_tokens=user_max_tokens, - user_api_base=user_api_base, - model=model, - route_type=route_type, - llm_router=llm_router, - ) + if skip_pre_call_logic: + logging_obj = self.data.get("litellm_logging_obj") + else: + self.data, logging_obj = await self.common_processing_pre_call_logic( + request=request, + general_settings=general_settings, + proxy_logging_obj=proxy_logging_obj, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_config=proxy_config, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + model=model, + route_type=route_type, + llm_router=llm_router, + ) tasks = [] # Start the moderation check (during_call_hook) as early as possible diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index e9c7cce0d73..055fdeb84f4 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -119,6 +119,34 @@ async def responses_api( f"Starting background response with polling for model={data.get('model')}" ) + # Run pre-call checks (rate limits, guardrails, budget) BEFORE creating + # polling ID. This ensures rate-limited requests get a synchronous 429 + # instead of a polling ID that immediately fails in the background task. + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + data, _logging_obj = await processor.common_processing_pre_call_logic( + request=request, + general_settings=general_settings, + proxy_logging_obj=proxy_logging_obj, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_config=proxy_config, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + route_type="aresponses", + llm_router=llm_router, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + # Initialize polling handler with configured TTL (from global config) polling_handler = ResponsePollingHandler( redis_cache=redis_usage_cache, @@ -134,7 +162,9 @@ async def responses_api( request_data=data, ) - # Start background task to stream and update cache + # Start background task to stream and update cache. + # Pass pre-processed data so the background task skips pre-call logic + # (rate limits, guardrails already checked above). asyncio.create_task( background_streaming_task( polling_id=polling_id, diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 7583f30eb2d..bcc98175773 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -65,7 +65,9 @@ async def background_streaming_task( # noqa: PLR0915 # Create processor processor = ProxyBaseLLMRequestProcessing(data=data) - # Make streaming request + # Make streaming request. + # Pre-call checks (rate limits, guardrails, budget) were already run + # before polling ID creation, so skip them here to avoid double-counting. response = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, @@ -83,6 +85,7 @@ async def background_streaming_task( # noqa: PLR0915 user_max_tokens=user_max_tokens, user_api_base=user_api_base, version=version, + skip_pre_call_logic=True, ) # Process streaming response following OpenAI events format diff --git a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py new file mode 100644 index 00000000000..b39f1bf43d2 --- /dev/null +++ b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py @@ -0,0 +1,104 @@ +""" +Unit tests for pre-call checks running before polling ID creation. + +Tests that rate limits, guardrails, and budget checks are enforced +BEFORE a polling ID is created, so rate-limited requests get a +synchronous error instead of a polling ID that immediately fails. +""" + +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import Request, Response + +sys.path.insert(0, os.path.abspath("../..")) + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + +class TestSkipPreCallLogic: + """Test that skip_pre_call_logic parameter works correctly""" + + @pytest.mark.asyncio + async def test_skip_pre_call_logic_skips_common_processing(self): + """When skip_pre_call_logic=True, common_processing_pre_call_logic should not be called""" + mock_logging_obj = MagicMock() + data = { + "model": "gpt-4", + "stream": True, + "litellm_logging_obj": mock_logging_obj, + } + processor = ProxyBaseLLMRequestProcessing(data=data) + + mock_proxy_logging = AsyncMock() + mock_proxy_logging.during_call_hook = AsyncMock() + + with ( + patch.object( + processor, "common_processing_pre_call_logic", new_callable=AsyncMock + ) as mock_pre_call, + patch( + "litellm.proxy.common_request_processing.route_request", + new_callable=AsyncMock, + return_value=MagicMock(), + ), + ): + try: + await processor.base_process_llm_request( + request=MagicMock(spec=Request), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + route_type="aresponses", + proxy_logging_obj=mock_proxy_logging, + llm_router=MagicMock(), + general_settings={}, + proxy_config=MagicMock(), + skip_pre_call_logic=True, + ) + except Exception: + pass # We only care that common_processing_pre_call_logic was not called + + mock_pre_call.assert_not_called() + + @pytest.mark.asyncio + async def test_without_skip_runs_common_processing(self): + """When skip_pre_call_logic=False (default), common_processing_pre_call_logic should be called""" + data = {"model": "gpt-4"} + processor = ProxyBaseLLMRequestProcessing(data=data) + + mock_logging_obj = MagicMock() + mock_proxy_logging = AsyncMock() + mock_proxy_logging.during_call_hook = AsyncMock() + + with ( + patch.object( + processor, + "common_processing_pre_call_logic", + new_callable=AsyncMock, + return_value=(data, mock_logging_obj), + ) as mock_pre_call, + patch( + "litellm.proxy.common_request_processing.route_request", + new_callable=AsyncMock, + ), + ): + try: + await processor.base_process_llm_request( + request=MagicMock(spec=Request), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + route_type="aresponses", + proxy_logging_obj=mock_proxy_logging, + llm_router=MagicMock(), + general_settings={}, + proxy_config=MagicMock(), + ) + except Exception: + pass + + mock_pre_call.assert_called_once() + + From c12717f494a5f7adbc7e2e0ab5f7cd814cc7c222 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 14:10:58 +0530 Subject: [PATCH 70/83] fix: address Greptile review comments - Guard logging_obj for None when skip_pre_call_logic=True: raise ValueError if litellm_logging_obj not in data, preventing AttributeError downstream - Add model=None to common_processing_pre_call_logic call in endpoints.py to match style of other call sites - Add test verifying rate-limited request never receives polling ID --- litellm/proxy/common_request_processing.py | 5 ++ .../proxy/response_api_endpoints/endpoints.py | 1 + .../test_response_polling_pre_call_checks.py | 63 ++++++++++++++++++- 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 84f9730a370..b86a7595ae9 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -912,6 +912,11 @@ class ProxyBaseLLMRequestProcessing: if skip_pre_call_logic: logging_obj = self.data.get("litellm_logging_obj") + if logging_obj is None: + raise ValueError( + "skip_pre_call_logic=True requires litellm_logging_obj to be set in data. " + "Ensure common_processing_pre_call_logic was called before using this parameter." + ) else: self.data, logging_obj = await self.common_processing_pre_call_logic( request=request, diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 055fdeb84f4..8023853e263 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -136,6 +136,7 @@ async def responses_api( user_request_timeout=user_request_timeout, user_max_tokens=user_max_tokens, user_api_base=user_api_base, + model=None, route_type="aresponses", llm_router=llm_router, ) diff --git a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py index b39f1bf43d2..cdea075d0d8 100644 --- a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py +++ b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py @@ -11,10 +11,11 @@ import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -from fastapi import Request, Response +from fastapi import HTTPException, Request, Response sys.path.insert(0, os.path.abspath("../..")) +import litellm from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -102,3 +103,63 @@ class TestSkipPreCallLogic: mock_pre_call.assert_called_once() +class TestPollingEndpointPreCallGuard: + """Test that the polling endpoint enforces pre-call checks before polling ID creation""" + + @pytest.mark.asyncio + async def test_rate_limit_error_prevents_polling_id_creation(self): + """When pre-call checks raise, generate_polling_id must not be called""" + from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler + + rate_limit_exc = litellm.RateLimitError( + message="TPM limit exceeded", + llm_provider="", + model="gpt-4", + ) + + generate_polling_id_mock = MagicMock(return_value="litellm_poll_test") + + with ( + patch.object( + ProxyBaseLLMRequestProcessing, + "common_processing_pre_call_logic", + new_callable=AsyncMock, + side_effect=rate_limit_exc, + ), + patch.object( + ProxyBaseLLMRequestProcessing, + "_handle_llm_api_exception", + new_callable=AsyncMock, + return_value=HTTPException(status_code=429, detail="Rate limit exceeded"), + ), + patch.object(ResponsePollingHandler, "generate_polling_id", generate_polling_id_mock), + ): + # Simulate the endpoint logic directly (avoids proxy_server import complexity) + data = {"model": "gpt-4", "background": True} + processor = ProxyBaseLLMRequestProcessing(data=data) + + raised_exc = None + try: + await processor.common_processing_pre_call_logic( + request=MagicMock(spec=Request), + general_settings={}, + proxy_logging_obj=AsyncMock(), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + version="1.0.0", + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=None, + route_type="aresponses", + llm_router=MagicMock(), + ) + except litellm.RateLimitError as e: + raised_exc = e + + # The exception was raised before generate_polling_id could be called + assert raised_exc is not None + generate_polling_id_mock.assert_not_called() + From 66f97a00a44d096c5ee0e54e9fbab59ea8ed9cd7 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 14:30:29 +0530 Subject: [PATCH 71/83] fix(test): rewrite polling pre-call guard test to call responses_api() directly Previously the test called common_processing_pre_call_logic in isolation, making generate_polling_id.assert_not_called() vacuously true. Now the test calls responses_api() end-to-end so it actually verifies that a rate-limited request never receives a polling ID. Co-Authored-By: Claude Sonnet 4.6 --- .../test_response_polling_pre_call_checks.py | 69 ++++++++++++------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py index cdea075d0d8..45e4e9e4d3e 100644 --- a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py +++ b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py @@ -108,7 +108,8 @@ class TestPollingEndpointPreCallGuard: @pytest.mark.asyncio async def test_rate_limit_error_prevents_polling_id_creation(self): - """When pre-call checks raise, generate_polling_id must not be called""" + """responses_api() must raise 429 and never call generate_polling_id when rate-limited""" + from litellm.proxy.response_api_endpoints.endpoints import responses_api from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler rate_limit_exc = litellm.RateLimitError( @@ -116,10 +117,37 @@ class TestPollingEndpointPreCallGuard: llm_provider="", model="gpt-4", ) - generate_polling_id_mock = MagicMock(return_value="litellm_poll_test") + proxy_server_patches = { + "litellm.proxy.proxy_server._read_request_body": AsyncMock( + return_value={"model": "gpt-4", "background": True} + ), + "litellm.proxy.proxy_server.general_settings": {}, + "litellm.proxy.proxy_server.llm_router": MagicMock(), + "litellm.proxy.proxy_server.native_background_mode": None, + "litellm.proxy.proxy_server.polling_cache_ttl": 3600, + "litellm.proxy.proxy_server.polling_via_cache_enabled": True, + "litellm.proxy.proxy_server.proxy_config": MagicMock(), + "litellm.proxy.proxy_server.proxy_logging_obj": AsyncMock(), + "litellm.proxy.proxy_server.redis_usage_cache": AsyncMock(), + "litellm.proxy.proxy_server.select_data_generator": None, + "litellm.proxy.proxy_server.user_api_base": None, + "litellm.proxy.proxy_server.user_max_tokens": None, + "litellm.proxy.proxy_server.user_model": None, + "litellm.proxy.proxy_server.user_request_timeout": None, + "litellm.proxy.proxy_server.user_temperature": None, + "litellm.proxy.proxy_server.version": "1.0.0", + } + with ( + patch.multiple("litellm.proxy.proxy_server", **{ + k.split(".")[-1]: v for k, v in proxy_server_patches.items() + }), + patch( + "litellm.proxy.response_polling.polling_handler.should_use_polling_for_request", + return_value=True, + ), patch.object( ProxyBaseLLMRequestProcessing, "common_processing_pre_call_logic", @@ -133,33 +161,22 @@ class TestPollingEndpointPreCallGuard: return_value=HTTPException(status_code=429, detail="Rate limit exceeded"), ), patch.object(ResponsePollingHandler, "generate_polling_id", generate_polling_id_mock), + # Prevent background task from running (avoids noise from incomplete mocks) + patch("asyncio.create_task"), + patch.object( + ResponsePollingHandler, + "create_initial_state", + new_callable=AsyncMock, + return_value=MagicMock(), + ), ): - # Simulate the endpoint logic directly (avoids proxy_server import complexity) - data = {"model": "gpt-4", "background": True} - processor = ProxyBaseLLMRequestProcessing(data=data) - - raised_exc = None - try: - await processor.common_processing_pre_call_logic( + with pytest.raises(HTTPException) as exc_info: + await responses_api( request=MagicMock(spec=Request), - general_settings={}, - proxy_logging_obj=AsyncMock(), + fastapi_response=MagicMock(spec=Response), user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), - version="1.0.0", - proxy_config=MagicMock(), - user_model=None, - user_temperature=None, - user_request_timeout=None, - user_max_tokens=None, - user_api_base=None, - model=None, - route_type="aresponses", - llm_router=MagicMock(), ) - except litellm.RateLimitError as e: - raised_exc = e - # The exception was raised before generate_polling_id could be called - assert raised_exc is not None - generate_polling_id_mock.assert_not_called() + assert exc_info.value.status_code == 429 + generate_polling_id_mock.assert_not_called() From 528daa8cf43767231531263f35fb6ad5a5ecadbf Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 14:44:01 +0530 Subject: [PATCH 72/83] feat(router): add per-model-group deployment affinity configuration Enable deployment_affinity, responses_api_deployment_check, and session_affinity to be configured per model group via router_settings.model_group_affinity_config, falling back to global settings for unconfigured groups. - Add model_group_affinity_config parameter to Router and DeploymentAffinityCheck - Add _get_effective_flags helper to resolve flags per model group - Update async_filter_deployments and async_pre_call_deployment_hook to use per-group config - Add 4 comprehensive tests covering per-group config, fallback, and override scenarios This allows fine-grained control of affinity behavior across model groups, e.g., enabling stickiness only for cross-provider deployments while leaving other groups free to load-balance. Co-Authored-By: Claude Haiku 4.5 --- litellm/router.py | 29 ++ .../deployment_affinity_check.py | 88 ++++-- litellm/types/router.py | 1 + .../test_deployment_affinity_check.py | 281 ++++++++++++++++++ 4 files changed, 371 insertions(+), 28 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 46998abb160..5fc0298cedb 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -301,6 +301,7 @@ class Router: RouterGeneralSettings ] = RouterGeneralSettings(), deployment_affinity_ttl_seconds: int = 3600, + model_group_affinity_config: Optional[Dict[str, List[str]]] = None, ignore_invalid_deployments: bool = False, ) -> None: """ @@ -641,6 +642,9 @@ class Router: self.model_group_retry_policy: Optional[ Dict[str, RetryPolicy] ] = model_group_retry_policy + self.model_group_affinity_config: Optional[ + Dict[str, List[str]] + ] = model_group_affinity_config self.allowed_fails_policy: Optional[AllowedFailsPolicy] = None if allowed_fails_policy is not None: @@ -661,6 +665,26 @@ class Router: if optional_pre_call_checks is not None: self.add_optional_pre_call_checks(optional_pre_call_checks) + # If model_group_affinity_config is set but no global affinity checks were + # enabled, we still need the DeploymentAffinityCheck callback (with global + # flags all False) so per-group config can activate affinity per model group. + if self.model_group_affinity_config and not any( + isinstance(cb, DeploymentAffinityCheck) + for cb in (self.optional_callbacks or []) + ): + if self.optional_callbacks is None: + self.optional_callbacks = [] + affinity_callback = DeploymentAffinityCheck( + cache=self.cache, + ttl_seconds=self.deployment_affinity_ttl_seconds, + enable_user_key_affinity=False, + enable_responses_api_affinity=False, + enable_session_id_affinity=False, + model_group_affinity_config=self.model_group_affinity_config, + ) + self.optional_callbacks.append(affinity_callback) + litellm.logging_callback_manager.add_litellm_callback(affinity_callback) + if self.alerting_config is not None: self._initialize_alerting() @@ -1311,6 +1335,10 @@ class Router: existing_affinity_callback.ttl_seconds = ( self.deployment_affinity_ttl_seconds ) + if self.model_group_affinity_config: + existing_affinity_callback.model_group_affinity_config = ( + self.model_group_affinity_config + ) else: affinity_callback = DeploymentAffinityCheck( cache=self.cache, @@ -1318,6 +1346,7 @@ class Router: enable_user_key_affinity=enable_user_key_affinity, enable_responses_api_affinity=enable_responses_api_affinity, enable_session_id_affinity=enable_session_id_affinity, + model_group_affinity_config=self.model_group_affinity_config, ) self.optional_callbacks.append(affinity_callback) litellm.logging_callback_manager.add_litellm_callback(affinity_callback) diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 8044f71d904..08da7d392d7 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -13,7 +13,7 @@ where routing to a consistent deployment is still beneficial. """ import hashlib -from typing import Any, Dict, List, Optional, cast +from typing import Any, Dict, List, Optional, Tuple, cast from typing_extensions import TypedDict @@ -46,6 +46,7 @@ class DeploymentAffinityCheck(CustomLogger): enable_user_key_affinity: bool, enable_responses_api_affinity: bool, enable_session_id_affinity: bool = False, + model_group_affinity_config: Optional[Dict[str, List[str]]] = None, ): super().__init__() self.cache = cache @@ -53,6 +54,32 @@ class DeploymentAffinityCheck(CustomLogger): self.enable_user_key_affinity = enable_user_key_affinity self.enable_responses_api_affinity = enable_responses_api_affinity self.enable_session_id_affinity = enable_session_id_affinity + self.model_group_affinity_config: Dict[str, List[str]] = ( + model_group_affinity_config or {} + ) + + def _get_effective_flags( + self, model_group: str + ) -> Tuple[bool, bool, bool]: + """ + Return (enable_user_key_affinity, enable_responses_api_affinity, enable_session_id_affinity) + for the given model group. + + If the model group has an explicit entry in model_group_affinity_config, use it. + Otherwise fall back to the global instance flags. + """ + group_checks = self.model_group_affinity_config.get(model_group) + if group_checks is not None: + return ( + "deployment_affinity" in group_checks, + "responses_api_deployment_check" in group_checks, + "session_affinity" in group_checks, + ) + return ( + self.enable_user_key_affinity, + self.enable_responses_api_affinity, + self.enable_session_id_affinity, + ) @staticmethod def _looks_like_sha256_hex(value: str) -> bool: @@ -277,8 +304,12 @@ class DeploymentAffinityCheck(CustomLogger): request_kwargs = request_kwargs or {} typed_healthy_deployments = cast(List[dict], healthy_deployments) + enable_user_key, enable_responses_api, enable_session_id = ( + self._get_effective_flags(model) + ) + # 1) Responses API continuity (high priority) - if self.enable_responses_api_affinity: + if enable_responses_api: previous_response_id = request_kwargs.get("previous_response_id") if previous_response_id is not None: responses_model_id = ( @@ -305,7 +336,7 @@ class DeploymentAffinityCheck(CustomLogger): return typed_healthy_deployments # 2) Session-id -> deployment affinity - if self.enable_session_id_affinity: + if enable_session_id: session_id = self._get_session_id_from_request_kwargs( request_kwargs=request_kwargs ) @@ -344,7 +375,7 @@ class DeploymentAffinityCheck(CustomLogger): ) # 3) User key -> deployment affinity - if not self.enable_user_key_affinity: + if not enable_user_key: return typed_healthy_deployments user_key = self._get_user_key_from_request_kwargs(request_kwargs=request_kwargs) @@ -394,22 +425,42 @@ class DeploymentAffinityCheck(CustomLogger): - LiteLLM runs async success callbacks via a background logging worker for performance. - We want affinity to be immediately available for subsequent requests. """ - if not self.enable_user_key_affinity and not self.enable_session_id_affinity: + metadata_dicts = self._iter_metadata_dicts(kwargs) + + # Extract deployment_model_name first — needed for both per-group flag resolution + # and cache key scoping. + deployment_model_name: Optional[str] = None + for metadata in metadata_dicts: + maybe_deployment_model_name = metadata.get("deployment_model_name") + if ( + isinstance(maybe_deployment_model_name, str) + and maybe_deployment_model_name + ): + deployment_model_name = maybe_deployment_model_name + break + + if not deployment_model_name: + return None + + # Resolve effective flags for this model group + enable_user_key, _enable_responses_api, enable_session_id = ( + self._get_effective_flags(deployment_model_name) + ) + + if not enable_user_key and not enable_session_id: return None user_key = None - if self.enable_user_key_affinity: + if enable_user_key: user_key = self._get_user_key_from_request_kwargs(request_kwargs=kwargs) session_id = None - if self.enable_session_id_affinity: + if enable_session_id: session_id = self._get_session_id_from_request_kwargs(request_kwargs=kwargs) if user_key is None and session_id is None: return None - metadata_dicts = self._iter_metadata_dicts(kwargs) - model_info = kwargs.get("model_info") if not isinstance(model_info, dict): model_info = None @@ -433,25 +484,6 @@ class DeploymentAffinityCheck(CustomLogger): ) return None - # Scope affinity by the Router deployment model name (alias-safe, consistent across - # heterogeneous providers, and matches standard logging's `model_map_key`). - deployment_model_name: Optional[str] = None - for metadata in metadata_dicts: - maybe_deployment_model_name = metadata.get("deployment_model_name") - if ( - isinstance(maybe_deployment_model_name, str) - and maybe_deployment_model_name - ): - deployment_model_name = maybe_deployment_model_name - break - - if not deployment_model_name: - verbose_router_logger.warning( - "DeploymentAffinityCheck: deployment_model_name missing; skipping affinity cache update. model_id=%s", - model_id, - ) - return None - if user_key is not None: try: cache_key = self.get_affinity_cache_key( diff --git a/litellm/types/router.py b/litellm/types/router.py index e8ff2115ff5..58411b1b85a 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -77,6 +77,7 @@ class UpdateRouterConfig(BaseModel): routing_strategy_args: Optional[dict] = None routing_strategy: Optional[str] = None model_group_retry_policy: Optional[dict] = None + model_group_affinity_config: Optional[Dict[str, List[str]]] = None allowed_fails: Optional[int] = None cooldown_time: Optional[float] = None num_retries: Optional[int] = None diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py index e500ad3ca6e..28311a30c0d 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py @@ -657,3 +657,284 @@ def test_cache_key_does_not_double_hash_user_api_key_hash(): user_key=user_api_key_hash, ) assert key.endswith(user_api_key_hash) + + +def test_get_effective_flags_returns_per_group_config(): + """ + _get_effective_flags should return per-group flags when the model group has an entry + in model_group_affinity_config, and global flags otherwise. + """ + callback = DeploymentAffinityCheck( + cache=AsyncMock(), + ttl_seconds=60, + enable_user_key_affinity=True, + enable_responses_api_affinity=True, + enable_session_id_affinity=False, + model_group_affinity_config={ + "gpt-4": ["deployment_affinity"], + "claude-3": ["session_affinity", "responses_api_deployment_check"], + }, + ) + + # gpt-4: only deployment_affinity + user_key, responses_api, session_id = callback._get_effective_flags("gpt-4") + assert user_key is True + assert responses_api is False + assert session_id is False + + # claude-3: session_affinity + responses_api_deployment_check + user_key, responses_api, session_id = callback._get_effective_flags("claude-3") + assert user_key is False + assert responses_api is True + assert session_id is True + + # unconfigured-model: falls back to global flags + user_key, responses_api, session_id = callback._get_effective_flags( + "unconfigured-model" + ) + assert user_key is True + assert responses_api is True + assert session_id is False + + +@pytest.mark.asyncio +async def test_model_group_affinity_config_only_applies_to_configured_group(): + """ + When model_group_affinity_config is set without global optional_pre_call_checks, + only configured model groups should get affinity behavior. + """ + mock_response_data = { + "id": "resp_mock-resp-per-group", + "object": "response", + "created_at": 1741476542, + "status": "completed", + "model": "openai/gpt-4", + "output": [ + { + "type": "message", + "id": "msg_pg", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Per-group response"}], + } + ], + "parallel_tool_calls": True, + "usage": {"input_tokens": 5, "output_tokens": 5, "total_tokens": 10}, + "text": {"format": {"type": "text"}}, + "error": None, + "previous_response_id": None, + } + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "azure/gpt-4-deploy-1", + "api_key": "mock-key-1", + "api_base": "https://mock-gpt4-1.openai.azure.com", + "api_version": "2024-02-01", + }, + "model_info": {"base_model": "gpt-4"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "azure/gpt-4-deploy-2", + "api_key": "mock-key-2", + "api_base": "https://mock-gpt4-2.openai.azure.com", + "api_version": "2024-02-01", + }, + "model_info": {"base_model": "gpt-4"}, + }, + { + "model_name": "claude-3", + "litellm_params": { + "model": "azure/claude-3-deploy-1", + "api_key": "mock-key-3", + "api_base": "https://mock-claude-1.openai.azure.com", + "api_version": "2024-02-01", + }, + "model_info": {"base_model": "claude-3"}, + }, + { + "model_name": "claude-3", + "litellm_params": { + "model": "azure/claude-3-deploy-2", + "api_key": "mock-key-4", + "api_base": "https://mock-claude-2.openai.azure.com", + "api_version": "2024-02-01", + }, + "model_info": {"base_model": "claude-3"}, + }, + ], + # No global optional_pre_call_checks — only per-group + model_group_affinity_config={ + "gpt-4": ["deployment_affinity"], + }, + ) + + user_api_key_hash = "test-per-group-key" + choice_calls = {"count": 0} + + def deterministic_choice(seq): + choice_calls["count"] += 1 + if choice_calls["count"] == 1: + return seq[0] + return seq[1] if len(seq) > 1 else seq[0] + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post, patch( + "litellm.router_strategy.simple_shuffle.random.choice", + side_effect=deterministic_choice, + ): + mock_post.return_value = MockResponse(mock_response_data, 200) + + # gpt-4: affinity should work — second request pinned to same deployment + first = await router.aresponses( + model="gpt-4", + input="Hello", + truncation="auto", + litellm_metadata={"user_api_key_hash": user_api_key_hash}, + ) + first_model_id = first._hidden_params["model_id"] + + second = await router.aresponses( + model="gpt-4", + input="Follow-up", + truncation="auto", + litellm_metadata={"user_api_key_hash": user_api_key_hash}, + ) + assert second._hidden_params["model_id"] == first_model_id + + # claude-3: no affinity configured — should NOT be pinned + choice_calls["count"] = 0 + first_claude = await router.aresponses( + model="claude-3", + input="Hello", + truncation="auto", + litellm_metadata={"user_api_key_hash": user_api_key_hash}, + ) + first_claude_id = first_claude._hidden_params["model_id"] + + second_claude = await router.aresponses( + model="claude-3", + input="Follow-up", + truncation="auto", + litellm_metadata={"user_api_key_hash": user_api_key_hash}, + ) + # With deterministic choice and len>1, second call picks seq[1] + assert second_claude._hidden_params["model_id"] != first_claude_id + + +@pytest.mark.asyncio +async def test_model_group_affinity_config_falls_back_to_global(): + """ + When both global optional_pre_call_checks and model_group_affinity_config are set, + unconfigured model groups should use the global settings. + """ + callback = DeploymentAffinityCheck( + cache=DualCache(), + ttl_seconds=60, + enable_user_key_affinity=True, + enable_responses_api_affinity=False, + enable_session_id_affinity=False, + model_group_affinity_config={ + "claude-3": ["session_affinity"], + }, + ) + + stable_model_map_key = "gpt-4" + user_key = "test-fallback-key" + + healthy_deployments = [ + { + "model_name": stable_model_map_key, + "litellm_params": {"model": "openai/gpt-4"}, + "model_info": {"id": "deployment-1"}, + }, + { + "model_name": stable_model_map_key, + "litellm_params": {"model": "openai/gpt-4"}, + "model_info": {"id": "deployment-2"}, + }, + ] + + # Set up affinity cache for gpt-4 (should work since global has deployment_affinity) + await callback.async_pre_call_deployment_hook( + kwargs={ + "model_info": {"id": "deployment-1"}, + "metadata": { + "user_api_key_hash": user_key, + "deployment_model_name": stable_model_map_key, + }, + }, + call_type=None, + ) + + # gpt-4 not in model_group_affinity_config, so global flags apply (user_key affinity ON) + filtered = await callback.async_filter_deployments( + model="gpt-4", + healthy_deployments=healthy_deployments, + messages=None, + request_kwargs={"metadata": {"user_api_key_hash": user_key}}, + parent_otel_span=None, + ) + assert len(filtered) == 1 + assert filtered[0]["model_info"]["id"] == "deployment-1" + + +@pytest.mark.asyncio +async def test_model_group_affinity_config_overrides_global(): + """ + When model_group_affinity_config specifies session_affinity for a model group, + user-key affinity (from global config) should NOT apply to that group. + """ + callback = DeploymentAffinityCheck( + cache=DualCache(), + ttl_seconds=60, + enable_user_key_affinity=True, + enable_responses_api_affinity=False, + enable_session_id_affinity=False, + model_group_affinity_config={ + "claude-3": ["session_affinity"], + }, + ) + + stable_model_map_key = "claude-3" + user_key = "test-override-key" + + healthy_deployments = [ + { + "model_name": stable_model_map_key, + "litellm_params": {"model": "anthropic/claude-3-opus"}, + "model_info": {"id": "deployment-1"}, + }, + { + "model_name": stable_model_map_key, + "litellm_params": {"model": "anthropic/claude-3-opus"}, + "model_info": {"id": "deployment-2"}, + }, + ] + + # Set up user-key affinity cache for claude-3 + cache_key = DeploymentAffinityCheck.get_affinity_cache_key( + model_group=stable_model_map_key, user_key=user_key + ) + await callback.cache.async_set_cache( + cache_key, {"model_id": "deployment-1"}, ttl=60 + ) + + # claude-3 has per-group config (session_affinity only), so user-key affinity + # should NOT apply even though it's globally enabled + filtered = await callback.async_filter_deployments( + model="claude-3", + healthy_deployments=healthy_deployments, + messages=None, + request_kwargs={"metadata": {"user_api_key_hash": user_key}}, + parent_otel_span=None, + ) + # All deployments returned (user-key affinity disabled for this group) + assert len(filtered) == 2 From 6af74f6594c9e8607251b11e17859b77ed2dff11 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 15:39:35 +0530 Subject: [PATCH 73/83] fix(router): restore debug log for missing deployment_model_name; warn on unknown affinity flags Co-Authored-By: Claude Haiku 4.5 --- .../pre_call_checks/deployment_affinity_check.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 08da7d392d7..728e689eaab 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -38,6 +38,9 @@ class DeploymentAffinityCheck(CustomLogger): """ CACHE_KEY_PREFIX = "deployment_affinity:v1" + VALID_FLAGS = frozenset( + {"deployment_affinity", "responses_api_deployment_check", "session_affinity"} + ) def __init__( self, @@ -57,6 +60,15 @@ class DeploymentAffinityCheck(CustomLogger): self.model_group_affinity_config: Dict[str, List[str]] = ( model_group_affinity_config or {} ) + for group, flags in self.model_group_affinity_config.items(): + unknown = set(flags) - self.VALID_FLAGS + if unknown: + verbose_router_logger.warning( + "DeploymentAffinityCheck: unknown flag(s) %s for model group '%s'; will be ignored. Valid flags: %s", + unknown, + group, + self.VALID_FLAGS, + ) def _get_effective_flags( self, model_group: str @@ -440,6 +452,9 @@ class DeploymentAffinityCheck(CustomLogger): break if not deployment_model_name: + verbose_router_logger.debug( + "DeploymentAffinityCheck: deployment_model_name missing in metadata; skipping affinity cache update." + ) return None # Resolve effective flags for this model group From a14122c28e5258d36339813539d084816a215cf0 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 15:50:43 +0530 Subject: [PATCH 74/83] docs: add per-model-group affinity configuration docs Co-Authored-By: Claude Haiku 4.5 --- docs/my-website/docs/proxy/config_settings.md | 1 + docs/my-website/docs/response_api.md | 79 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index f5b611a85a6..d7d6079c517 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -363,6 +363,7 @@ router_settings: | router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) | | optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity`, `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` | | deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). | +| model_group_affinity_config | Dict[str, List[str]] | Per-model-group affinity flags. Keys are model group names; values are lists of checks to enable (`deployment_affinity`, `responses_api_deployment_check`, `session_affinity`). Groups not listed fall back to the global `optional_pre_call_checks`. [Docs](../response_api.md#per-model-group-affinity-configuration) | | ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. | | search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search/index.md) | | guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) | diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index fb55ae9f9d0..56b8170995c 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -1364,6 +1364,85 @@ litellm --config config.yaml | `deployment_affinity` | Simple sticky sessions | All requests from same API key | ❌ Reduces quota by # of users | +## Per-Model-Group Affinity Configuration + +By default, `optional_pre_call_checks` applies globally to all model groups. Use `model_group_affinity_config` when you want different affinity behavior per model group — for example, enabling stickiness only for models spread across providers (Azure + Bedrock) while leaving single-provider groups free to load-balance. + +Groups not listed fall back to the global `optional_pre_call_checks` settings. + + + + +```python +router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "azure/gpt-4", "api_key": "...", "api_base": "https://endpoint1.openai.azure.com"}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "bedrock/anthropic.claude-v2", "aws_region_name": "us-east-1"}, + }, + { + "model_name": "text-embedding-ada-002", + "litellm_params": {"model": "azure/text-embedding-ada-002", "api_key": "...", "api_base": "https://endpoint1.openai.azure.com"}, + }, + { + "model_name": "text-embedding-ada-002", + "litellm_params": {"model": "azure/text-embedding-ada-002", "api_key": "...", "api_base": "https://endpoint2.openai.azure.com"}, + }, + ], + # gpt-4: cross-provider (Azure + Bedrock) — enable deployment affinity + # text-embedding-ada-002: same provider — no affinity, let it load balance freely + model_group_affinity_config={ + "gpt-4": ["deployment_affinity", "responses_api_deployment_check"], + }, +) +``` + + + + +```yaml title="config.yaml" +model_list: + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4 + api_key: os.environ/AZURE_API_KEY_1 + api_base: https://endpoint1.openai.azure.com + + - model_name: gpt-4 + litellm_params: + model: bedrock/anthropic.claude-v2 + aws_region_name: us-east-1 + + - model_name: text-embedding-ada-002 + litellm_params: + model: azure/text-embedding-ada-002 + api_key: os.environ/AZURE_API_KEY_1 + api_base: https://endpoint1.openai.azure.com + + - model_name: text-embedding-ada-002 + litellm_params: + model: azure/text-embedding-ada-002 + api_key: os.environ/AZURE_API_KEY_2 + api_base: https://endpoint2.openai.azure.com + +router_settings: + # gpt-4: cross-provider — enable stickiness + # text-embedding-ada-002: not listed — load balances freely + model_group_affinity_config: + "gpt-4": + - deployment_affinity + - responses_api_deployment_check +``` + + + + +**Supported values:** `deployment_affinity`, `responses_api_deployment_check`, `session_affinity` + ## Calling non-Responses API endpoints (`/responses` to `/chat/completions` Bridge) LiteLLM allows you to call non-Responses API models via a bridge to LiteLLM's `/chat/completions` endpoint. This is useful for calling Anthropic, Gemini and even non-Responses API OpenAI models. From ec7c94c61f680b8e769416cd89e7ca2f50d0c6d1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Mar 2026 16:39:18 +0530 Subject: [PATCH 75/83] Add about managed ids in the doc --- docs/my-website/docs/tutorials/file_search_responses_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/tutorials/file_search_responses_api.md b/docs/my-website/docs/tutorials/file_search_responses_api.md index 9df18b4095e..f539d11c6d1 100644 --- a/docs/my-website/docs/tutorials/file_search_responses_api.md +++ b/docs/my-website/docs/tutorials/file_search_responses_api.md @@ -19,7 +19,7 @@ Two paths are covered: | **Native passthrough** | Provider natively supports `file_search` (OpenAI, Azure) | Decodes unified vector store ID → forwards to provider as-is | | **Emulated fallback** | Provider doesn't support `file_search` (Anthropic, Bedrock, etc.) | Converts to a function tool → intercepts tool call → runs vector search → synthesizes OpenAI-format output | - +In `tools[].vector_store_ids`, LiteLLM accepts both provider-native IDs (e.g. `vs_...`) **and** **managed vector store unified IDs** (URL-safe base64 strings from the proxy managed-vector flow), e.g. `litellm.responses(..., tools=[{"type": "file_search", "vector_store_ids": ["bGl0ZWxsbV9wcm94eT..."]}])`. ## Usage From 4d06b1cf0a3fa3d26f40d729d92ff261de3454f3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Mar 2026 16:48:27 +0530 Subject: [PATCH 76/83] fix doc --- .../tutorials/file_search_responses_api.md | 39 +++++++------------ docs/my-website/sidebars.js | 2 +- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/docs/my-website/docs/tutorials/file_search_responses_api.md b/docs/my-website/docs/tutorials/file_search_responses_api.md index f539d11c6d1..d74bdf9cb83 100644 --- a/docs/my-website/docs/tutorials/file_search_responses_api.md +++ b/docs/my-website/docs/tutorials/file_search_responses_api.md @@ -15,7 +15,7 @@ LiteLLM keeps one OpenAI-compatible output shape while routing requests through Two paths are covered: | Path | When it runs | What LiteLLM does | -|||| +| --- | --- | --- | | **Native passthrough** | Provider natively supports `file_search` (OpenAI, Azure) | Decodes unified vector store ID → forwards to provider as-is | | **Emulated fallback** | Provider doesn't support `file_search` (Anthropic, Bedrock, etc.) | Converts to a function tool → intercepts tool call → runs vector search → synthesizes OpenAI-format output | @@ -102,7 +102,7 @@ print(response.output) ### Behavior Matrix | Path | SDK model | Proxy model | Behavior | -||||| +| --- | --- | --- | --- | | Native passthrough | `openai/gpt-4.1` | `gpt-4.1` | Provider executes native `file_search` | | Emulated fallback | `anthropic/claude-sonnet-4-5` | `claude-sonnet` | LiteLLM converts to function tool and synthesizes OpenAI-format output | @@ -220,30 +220,17 @@ validate_file_search_response(response) ## Q&A -### Q: Why do I see `UnsupportedParamsError`? - -A: This usually means `file_search` was passed to a provider that does not support it natively and emulation could not route correctly. -Check: -- The model string is valid (for example, `anthropic/claude-sonnet-4-5`). -- `custom_llm_provider` resolves correctly so LiteLLM can load the provider config. - -### Q: Why does vector search return no results? - -A: Common causes: -- The vector store ID is wrong or has no files attached. -- In LiteLLM-managed stores, file ingestion is not complete (`status != completed`). -- The query is too narrow; try a broader query. - -### Q: Why am I getting `403 Access denied` on vector store calls? - -A: The caller does not have access to that vector store. -- The store may belong to another team. -- Use an admin/proxy key if your setup requires cross-team access. - -### Q: Why are `annotations` empty in emulated mode? - -A: `file_citation` annotations require `file_id` metadata in search results. -If your vector backend does not return file-level metadata, the answer text is still generated but citations can be empty. +- **Why do I see `UnsupportedParamsError`?** This usually means `file_search` was passed to a provider that does not support it natively and emulation could not route correctly. Check: + - The model string is valid (for example, `anthropic/claude-sonnet-4-5`). + - `custom_llm_provider` resolves correctly so LiteLLM can load the provider config. +- **Why does vector search return no results?** Common causes: + - The vector store ID is wrong or has no files attached. + - In LiteLLM-managed stores, file ingestion is not complete (`status != completed`). + - The query is too narrow; try a broader query. +- **Why am I getting `403 Access denied` on vector store calls?** The caller does not have access to that vector store. + - The store may belong to another team. + - Use an admin/proxy key if your setup requires cross-team access. +- **Why are `annotations` empty in emulated mode?** `file_citation` annotations require `file_id` metadata in search results. If your vector backend does not return file-level metadata, the answer text is still generated but citations can be empty. diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 046c5602fcb..df6cd939b78 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -1433,7 +1433,7 @@ const learnSidebar = { }, items: [ "tutorials/prompt_caching", - "tutorials/file_search_responses_api" + "tutorials/file_search_responses_api", "tutorials/anthropic_file_usage", "tutorials/gemini_realtime_with_audio", "tutorials/litellm_proxy_aporia", From 2a69426e2f8dcca0b283691f9df78ba3df8f6e30 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Mar 2026 23:18:58 +0530 Subject: [PATCH 77/83] Fix mypy and code qa issues --- .../pass_through_endpoints.py | 201 +++++++++--------- .../responses/file_search/emulated_handler.py | 88 +++++--- litellm/responses/main.py | 188 +++++++++------- 3 files changed, 277 insertions(+), 200 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 0f676a1feb2..e2f7646c0aa 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -40,6 +40,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.passthrough import BasePassthroughUtils from litellm.proxy._types import ( + CommonProxyErrors, ConfigFieldInfo, ConfigFieldUpdate, LiteLLMRoutes, @@ -2155,6 +2156,102 @@ def _get_combined_pass_through_endpoints( return pass_through_endpoints + config_pass_through_endpoints +async def _register_pass_through_endpoint( + endpoint: Union[Dict[str, Any], PassThroughGenericEndpoint], + app: FastAPI, + premium_user: bool, + visited_endpoints: set[str], +) -> None: + endpoint_data: Dict[str, Any] + if isinstance(endpoint, PassThroughGenericEndpoint): + endpoint_data = endpoint.model_dump() + else: + endpoint_data = endpoint + + if endpoint_data.get("id") is None: + endpoint_data["id"] = str(uuid.uuid4()) + endpoint_id = cast(str, endpoint_data["id"]) + + target = endpoint_data.get("target") + path = endpoint_data.get("path") + if path is None: + raise ValueError("Path is required for pass-through endpoint") + + custom_headers = await set_env_variables_in_header( + custom_headers=endpoint_data.get("headers") + ) + forward_headers = endpoint_data.get("forward_headers") + merge_query_params = endpoint_data.get("merge_query_params") + default_query_params = endpoint_data.get("default_query_params") + auth = endpoint_data.get("auth") + dependencies = None + + if auth is not None and str(auth).lower() == "true": + if premium_user is not True: + raise ValueError( + "Error Setting Authentication on Pass Through Endpoint: {}".format( + CommonProxyErrors.not_premium_user.value + ) + ) + dependencies = [Depends(user_api_key_auth)] + if path not in LiteLLMRoutes.openai_routes.value: + LiteLLMRoutes.openai_routes.value.append(path) + + if target is None: + return + + guardrails = endpoint_data.get("guardrails") + methods = endpoint_data.get("methods") + cost_per_request = endpoint_data.get("cost_per_request") + + verbose_proxy_logger.debug( + "Initializing pass through endpoint: %s (ID: %s)", path, endpoint_id + ) + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path=path, + target=target, + custom_headers=custom_headers, + forward_headers=forward_headers, + merge_query_params=merge_query_params, + dependencies=dependencies, + cost_per_request=cost_per_request, + endpoint_id=endpoint_id, + guardrails=guardrails, + methods=methods, + default_query_params=default_query_params, + ) + + methods_for_key = methods if methods else ["GET", "POST", "PUT", "DELETE", "PATCH"] + methods_str = ",".join(sorted(methods_for_key)) + visited_endpoints.add(f"{endpoint_id}:exact:{path}:{methods_str}") + + if endpoint_data.get("include_subpath", False) is True: + if auth is not None and str(auth).lower() == "true": + wildcard_path = path.rstrip("/") + "/*" + if wildcard_path not in LiteLLMRoutes.openai_routes.value: + LiteLLMRoutes.openai_routes.value.append(wildcard_path) + InitPassThroughEndpointHelpers.add_subpath_route( + app=app, + path=path, + target=target, + custom_headers=custom_headers, + forward_headers=forward_headers, + merge_query_params=merge_query_params, + dependencies=dependencies, + cost_per_request=cost_per_request, + endpoint_id=endpoint_id, + guardrails=guardrails, + methods=methods, + default_query_params=default_query_params, + ) + visited_endpoints.add(f"{endpoint_id}:subpath:{path}:{methods_str}") + + verbose_proxy_logger.debug( + "Added new pass through endpoint: %s (ID: %s)", path, endpoint_id + ) + + async def initialize_pass_through_endpoints( pass_through_endpoints: Union[List[Dict], List[PassThroughGenericEndpoint]], ): @@ -2171,10 +2268,7 @@ async def initialize_pass_through_endpoints( Returns: None """ - from litellm._uuid import uuid - verbose_proxy_logger.debug("initializing pass through endpoints") - from litellm.proxy._types import CommonProxyErrors, LiteLLMRoutes from litellm.proxy.proxy_server import ( app, config_passthrough_endpoints, @@ -2201,105 +2295,14 @@ async def initialize_pass_through_endpoints( InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() ) - visited_endpoints = set() + visited_endpoints: set[str] = set() for endpoint in combined_pass_through_endpoints: - if isinstance(endpoint, PassThroughGenericEndpoint): - endpoint = endpoint.model_dump() - - # Auto-generate ID for backwards compatibility if not present - if endpoint.get("id") is None: - endpoint["id"] = str(uuid.uuid4()) - - # Get the endpoint_id as a string (guaranteed to be set at this point) - endpoint_id: str = endpoint["id"] - - _target = endpoint.get("target", None) - _path: Optional[str] = endpoint.get("path", None) - if _path is None: - raise ValueError("Path is required for pass-through endpoint") - _custom_headers = endpoint.get("headers", None) - _custom_headers = await set_env_variables_in_header( - custom_headers=_custom_headers - ) - _forward_headers = endpoint.get("forward_headers", None) - _merge_query_params = endpoint.get("merge_query_params", None) - _default_query_params = endpoint.get("default_query_params", None) - _auth = endpoint.get("auth", None) - _dependencies = None - if _auth is not None and str(_auth).lower() == "true": - if premium_user is not True: - raise ValueError( - "Error Setting Authentication on Pass Through Endpoint: {}".format( - CommonProxyErrors.not_premium_user.value - ) - ) - _dependencies = [Depends(user_api_key_auth)] - if _path not in LiteLLMRoutes.openai_routes.value: - LiteLLMRoutes.openai_routes.value.append(_path) - - if _target is None: - continue - - # Get guardrails config if present - _guardrails = endpoint.get("guardrails", None) - - # Get methods list if present (None means all methods for backward compatibility) - _methods = endpoint.get("methods", None) - - # Add exact path route - verbose_proxy_logger.debug( - "Initializing pass through endpoint: %s (ID: %s)", _path, endpoint_id - ) - InitPassThroughEndpointHelpers.add_exact_path_route( + await _register_pass_through_endpoint( + endpoint=endpoint, app=app, - path=_path, - target=_target, - custom_headers=_custom_headers, - forward_headers=_forward_headers, - merge_query_params=_merge_query_params, - dependencies=_dependencies, - cost_per_request=endpoint.get("cost_per_request", None), - endpoint_id=endpoint_id, - guardrails=_guardrails, - methods=_methods, - default_query_params=_default_query_params, - ) - - # Generate route key with methods for tracking - methods_for_key = ( - _methods if _methods else ["GET", "POST", "PUT", "DELETE", "PATCH"] - ) - methods_str = ",".join(sorted(methods_for_key)) - visited_endpoints.add(f"{endpoint_id}:exact:{_path}:{methods_str}") - - # Add wildcard route for sub-paths - if endpoint.get("include_subpath", False) is True: - # Register wildcard path in openai_routes so non-admin users - # can access subpath routes when auth is enabled - if _auth is not None and str(_auth).lower() == "true": - _wildcard_path = _path.rstrip("/") + "/*" - if _wildcard_path not in LiteLLMRoutes.openai_routes.value: - LiteLLMRoutes.openai_routes.value.append(_wildcard_path) - InitPassThroughEndpointHelpers.add_subpath_route( - app=app, - path=_path, - target=_target, - custom_headers=_custom_headers, - forward_headers=_forward_headers, - merge_query_params=_merge_query_params, - dependencies=_dependencies, - cost_per_request=endpoint.get("cost_per_request", None), - endpoint_id=endpoint_id, - guardrails=_guardrails, - methods=_methods, - default_query_params=_default_query_params, - ) - - visited_endpoints.add(f"{endpoint_id}:subpath:{_path}:{methods_str}") - - verbose_proxy_logger.debug( - "Added new pass through endpoint: %s (ID: %s)", _path, endpoint_id + premium_user=premium_user, + visited_endpoints=visited_endpoints, ) # remove the ones that are not visited from the list diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index f770648e639..4700d673998 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -14,10 +14,10 @@ Flow: import json import time import uuid -from typing import Any, Dict, Iterable, List, Optional, Tuple, cast +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union, cast from litellm._logging import verbose_logger -from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.llms.openai import ResponseOutputItem, ResponsesAPIResponse from litellm.types.vector_stores import VectorStoreSearchResult # Keep ToolParam broad so we stay compatible with both dict and Pydantic forms @@ -30,6 +30,7 @@ FILE_SEARCH_FUNCTION_NAME = "litellm_file_search" # Detection # --------------------------------------------------------------------------- + def should_use_emulated_file_search( tools: Optional[Iterable[ToolParam]], provider_config: Any, # BaseResponsesAPIConfig @@ -37,9 +38,7 @@ def should_use_emulated_file_search( """Return True when there is a file_search tool and the provider can't handle it natively.""" if not tools: return False - has_fs = any( - isinstance(t, dict) and t.get("type") == "file_search" for t in tools - ) + has_fs = any(isinstance(t, dict) and t.get("type") == "file_search" for t in tools) if not has_fs: return False return provider_config is None or not provider_config.supports_native_file_search() @@ -49,6 +48,7 @@ def should_use_emulated_file_search( # Tool conversion # --------------------------------------------------------------------------- + def _build_function_tool(vector_store_ids: List[str]) -> Dict[str, Any]: """ Create a Responses API function-tool definition that describes file search. @@ -104,7 +104,7 @@ def _replace_file_search_tools( non_file_search: List[Dict[str, Any]] = [] vector_store_ids: List[str] = [] - for tool in (tools or []): + for tool in tools or []: if isinstance(tool, dict) and tool.get("type") == "file_search": ids = tool.get("vector_store_ids") or [] vector_store_ids.extend(ids) @@ -123,6 +123,7 @@ def _replace_file_search_tools( # Search execution # --------------------------------------------------------------------------- + async def _run_vector_searches( queries: List[str], vector_store_ids: List[str], @@ -150,7 +151,11 @@ async def _run_vector_searches( vector_store_id=vs_id, query=query, ) - results_data = response.get("data") if isinstance(response, dict) else getattr(response, "data", None) + results_data = ( + response.get("data") + if isinstance(response, dict) + else getattr(response, "data", None) + ) if results_data: all_results.extend(results_data) except Exception as exc: @@ -168,6 +173,7 @@ async def _run_vector_searches( # Result formatting # --------------------------------------------------------------------------- + def _get_field(result: Any, key: str, default: Any = None) -> Any: """Read a field from either a dict/TypedDict or an attribute-based object.""" if isinstance(result, dict): @@ -319,13 +325,27 @@ def _build_message_output( def _extract_text_from_responses_output(response: ResponsesAPIResponse) -> str: """Pull the assistant's text from the provider's response.""" for item in response.output: - item_type = item.get("type") if isinstance(item, dict) else getattr(item, "type", None) + item_type = ( + item.get("type") if isinstance(item, dict) else getattr(item, "type", None) + ) if item_type == "message": - content = item.get("content") if isinstance(item, dict) else getattr(item, "content", []) - for block in (content or []): - block_type = block.get("type") if isinstance(block, dict) else getattr(block, "type", None) + content = ( + item.get("content") + if isinstance(item, dict) + else getattr(item, "content", []) + ) + for block in content or []: + block_type = ( + block.get("type") + if isinstance(block, dict) + else getattr(block, "type", None) + ) if block_type == "output_text": - raw = block.get("text") if isinstance(block, dict) else getattr(block, "text", "") + raw = ( + block.get("text") + if isinstance(block, dict) + else getattr(block, "text", "") + ) return str(raw) if raw is not None else "" return "" @@ -345,13 +365,16 @@ def _synthesize_responses_api_response( synthesized _hidden_params so that billing callbacks see the total cost of both provider calls that the emulated flow makes. """ + synthesized_output: List[Dict[str, Any]] = [file_search_call_output, message_output] synthesized = ResponsesAPIResponse( id=getattr(original_response, "id", f"resp_{uuid.uuid4().hex}"), object="response", created_at=getattr(original_response, "created_at", int(time.time())), status="completed", model=getattr(original_response, "model", ""), - output=[file_search_call_output, message_output], + output=cast( + List[Union[ResponseOutputItem, Dict[str, Any]]], synthesized_output + ), usage=getattr(original_response, "usage", None), error=None, ) @@ -359,9 +382,15 @@ def _synthesize_responses_api_response( hidden = dict(getattr(original_response, "_hidden_params") or {}) if first_response is not None and hasattr(first_response, "_hidden_params"): first_hidden = getattr(first_response, "_hidden_params") or {} - first_cost = first_hidden.get("response_cost") if isinstance(first_hidden, dict) else getattr(first_hidden, "response_cost", None) + first_cost = ( + first_hidden.get("response_cost") + if isinstance(first_hidden, dict) + else getattr(first_hidden, "response_cost", None) + ) if first_cost is not None: - current_cost = hidden.get("response_cost") if isinstance(hidden, dict) else 0 + current_cost = ( + hidden.get("response_cost") if isinstance(hidden, dict) else 0 + ) hidden["response_cost"] = (current_cost or 0) + first_cost synthesized._hidden_params = hidden return synthesized @@ -371,8 +400,12 @@ def _synthesize_responses_api_response( # Main entry point # --------------------------------------------------------------------------- -async def _call_aresponses(input, model, tools, **kwargs): # pragma: no cover – thin wrapper for patching in tests + +async def _call_aresponses( + input, model, tools, **kwargs +): # pragma: no cover – thin wrapper for patching in tests from litellm.responses.main import aresponses + return await aresponses(input=input, model=model, tools=tools, **kwargs) @@ -458,10 +491,17 @@ async def aresponses_with_emulated_file_search( for tool_call in file_search_calls: if isinstance(tool_call, dict): - call_id = tool_call.get("call_id") or tool_call.get("id") or file_search_call_id + call_id = str( + tool_call.get("call_id") or tool_call.get("id") or file_search_call_id + ) raw_args = tool_call.get("arguments") or "{}" else: - call_id = getattr(tool_call, "call_id", None) or getattr(tool_call, "id", file_search_call_id) + raw_call_id = ( + getattr(tool_call, "call_id", None) + or getattr(tool_call, "id", None) + or file_search_call_id + ) + call_id = str(raw_call_id) raw_args = getattr(tool_call, "arguments", "{}") or "{}" try: @@ -500,7 +540,11 @@ async def aresponses_with_emulated_file_search( # Including all output items (text blocks, reasoning, non-file-search calls) ensures providers # like Anthropic that emit text before the tool call have complete conversation context. # Serialize Pydantic model instances to plain dicts so the transformation layer can call .get(). - original_input_items = list(input) if isinstance(input, (list, tuple)) else [{"role": "user", "content": str(input)}] + original_input_items = ( + list(input) + if isinstance(input, (list, tuple)) + else [{"role": "user", "content": str(input)}] + ) first_response_output_items: List[Any] = [] for _item in first_response.output: if isinstance(_item, dict): @@ -510,11 +554,7 @@ async def aresponses_with_emulated_file_search( else: first_response_output_items.append(_item) - follow_up_input = ( - original_input_items - + first_response_output_items - + tool_results - ) + follow_up_input = original_input_items + first_response_output_items + tool_results # 6. Follow-up call — provider writes the final answer given search results. # Also an internal sub-call; billing is suppressed so the outer call fires once. diff --git a/litellm/responses/main.py b/litellm/responses/main.py index c429cfdbc39..16f97773fc6 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -77,9 +77,7 @@ def _has_file_search_tool(tools: Optional[Any]) -> bool: """Return True if any tool in the list has type 'file_search'.""" if not tools: return False - return any( - isinstance(t, dict) and t.get("type") == "file_search" for t in tools - ) + return any(isinstance(t, dict) and t.get("type") == "file_search" for t in tools) def mock_responses_api_response( @@ -486,7 +484,9 @@ async def aresponses( prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) original_model = model - if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and litellm_logging_obj.should_run_prompt_management_hooks( + if isinstance( + litellm_logging_obj, LiteLLMLoggingObj + ) and litellm_logging_obj.should_run_prompt_management_hooks( prompt_id=prompt_id, non_default_params=kwargs ): if isinstance(input, str): @@ -514,9 +514,7 @@ async def aresponses( ) input = cast(Union[str, ResponseInputParam], merged_input) if model != original_model: - _, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model - ) + _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model) kwargs.pop("prompt_id", None) kwargs["_async_prompt_merged_params"] = merged_optional_params @@ -588,6 +586,88 @@ async def aresponses( ) +def _apply_prompt_management_to_responses_call( + input: Union[str, ResponseInputParam], + model: str, + custom_llm_provider: Optional[str], + litellm_logging_obj: Optional[LiteLLMLoggingObj], + kwargs: Dict[str, Any], + local_vars: Dict[str, Any], +) -> tuple[Union[str, ResponseInputParam], str, Optional[str]]: + async_merged = kwargs.pop("_async_prompt_merged_params", None) + if async_merged is not None: + for key, value in async_merged.items(): + local_vars[key] = value + return input, model, custom_llm_provider + + prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) + prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) + original_model = model + + if isinstance(input, str): + client_input: List[AllMessageValues] = [{"role": "user", "content": input}] + else: + client_input = [ + item # type: ignore[misc] + for item in input + if isinstance(item, dict) and "role" in item + ] + + if isinstance( + litellm_logging_obj, LiteLLMLoggingObj + ) and litellm_logging_obj.should_run_prompt_management_hooks( + prompt_id=prompt_id, non_default_params=kwargs + ): + ( + model, + merged_input, + merged_optional_params, + ) = litellm_logging_obj.get_chat_completion_prompt( + model=model, + messages=client_input, + non_default_params=kwargs, + prompt_id=prompt_id, + prompt_variables=prompt_variables, + prompt_label=kwargs.get("prompt_label", None), + prompt_version=kwargs.get("prompt_version", None), + ) + input = cast(Union[str, ResponseInputParam], merged_input) + local_vars["input"] = input + local_vars["model"] = model + if model != original_model: + _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model) + local_vars["custom_llm_provider"] = custom_llm_provider + for key, value in merged_optional_params.items(): + local_vars[key] = value + + return input, model, custom_llm_provider + + +def _resolve_model_provider_for_responses( + model: str, + custom_llm_provider: Optional[str], + litellm_params: GenericLiteLLMParams, + local_vars: Dict[str, Any], +) -> tuple[str, Optional[str]]: + ( + model, + custom_llm_provider, + dynamic_api_key, + dynamic_api_base, + ) = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + local_vars["custom_llm_provider"] = custom_llm_provider + if dynamic_api_key is not None: + litellm_params.api_key = dynamic_api_key + if dynamic_api_base is not None: + litellm_params.api_base = dynamic_api_base + return model, custom_llm_provider + + @client def responses( input: Union[str, ResponseInputParam], @@ -659,80 +739,27 @@ def responses( mock_response=litellm_params.mock_response ) - ( - model, - custom_llm_provider, - dynamic_api_key, - dynamic_api_base, - ) = litellm.get_llm_provider( + model, custom_llm_provider = _resolve_model_provider_for_responses( model=model, custom_llm_provider=custom_llm_provider, - api_base=litellm_params.api_base, - api_key=litellm_params.api_key, + litellm_params=litellm_params, + local_vars=local_vars, ) - # Update local_vars with detected provider (fixes #19782) - local_vars["custom_llm_provider"] = custom_llm_provider - - # Use dynamic credentials from get_llm_provider (e.g., when use_litellm_proxy=True) - if dynamic_api_key is not None: - litellm_params.api_key = dynamic_api_key - if dynamic_api_base is not None: - litellm_params.api_base = dynamic_api_base - ######################################################### # PROMPT MANAGEMENT # If aresponses() already ran the async hook, it pops prompt_id and # passes the result via _async_prompt_merged_params — apply those # directly and skip the sync hook to avoid double-merging. ######################################################### - _async_merged = kwargs.pop("_async_prompt_merged_params", None) - if _async_merged is not None: - for k, v in _async_merged.items(): - local_vars[k] = v - else: - prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) - prompt_variables = cast( - Optional[dict], kwargs.get("prompt_variables", None) - ) - original_model = model - - if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and litellm_logging_obj.should_run_prompt_management_hooks( - prompt_id=prompt_id, non_default_params=kwargs - ): - if isinstance(input, str): - client_input: List[AllMessageValues] = [ - {"role": "user", "content": input} - ] - else: - client_input = [ - item # type: ignore[misc] - for item in input - if isinstance(item, dict) and "role" in item - ] - ( - model, - merged_input, - merged_optional_params, - ) = litellm_logging_obj.get_chat_completion_prompt( - model=model, - messages=client_input, - non_default_params=kwargs, - prompt_id=prompt_id, - prompt_variables=prompt_variables, - prompt_label=kwargs.get("prompt_label", None), - prompt_version=kwargs.get("prompt_version", None), - ) - input = cast(Union[str, ResponseInputParam], merged_input) - local_vars["input"] = input - local_vars["model"] = model - if model != original_model: - _, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model - ) - local_vars["custom_llm_provider"] = custom_llm_provider - for k, v in merged_optional_params.items(): - local_vars[k] = v + input, model, custom_llm_provider = _apply_prompt_management_to_responses_call( + input=input, + model=model, + custom_llm_provider=custom_llm_provider, + litellm_logging_obj=litellm_logging_obj, + kwargs=kwargs, + local_vars=local_vars, + ) ######################################################### # Update input and tools with provider-specific file IDs if managed files are used @@ -803,12 +830,16 @@ def responses( return run_async_function(aresponses_api_with_mcp, **mcp_call_kwargs) # get provider config - responses_api_provider_config: Optional[ - BaseResponsesAPIConfig - ] = ProviderConfigManager.get_provider_responses_api_config( - model=model, - provider=custom_llm_provider, - ) + responses_api_provider_config: Optional[BaseResponsesAPIConfig] + if custom_llm_provider is None: + responses_api_provider_config = None + else: + responses_api_provider_config = ( + ProviderConfigManager.get_provider_responses_api_config( + model=model, + provider=custom_llm_provider, + ) + ) local_vars.update(kwargs) # Map reasoning_effort (from litellm_params/proxy config) to reasoning when not set @@ -919,6 +950,9 @@ def responses( ) # Call the handler with _is_async flag instead of directly calling the async handler + if custom_llm_provider is None: + raise ValueError("custom_llm_provider is required but passed as None") + response = base_llm_http_handler.response_api_handler( model=model, input=input, From 92e98a2fd5d46a8a7b1c407ecff84e926914768d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Mar 2026 23:19:39 +0530 Subject: [PATCH 78/83] Fix test_aaamodel_prices_and_context_window_json_is_valid --- tests/test_litellm/test_utils.py | 230 ++++++++++++++++++++----------- 1 file changed, 152 insertions(+), 78 deletions(-) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 64488e2fb6a..38b7b576d4f 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -38,22 +38,28 @@ def test_check_provider_match_azure_ai_allows_openai_and_azure(): This is needed for Azure Model Router which can route to OpenAI models. """ # azure_ai should match openai models - assert _check_provider_match( - model_info={"litellm_provider": "openai"}, - custom_llm_provider="azure_ai" - ) is True + assert ( + _check_provider_match( + model_info={"litellm_provider": "openai"}, custom_llm_provider="azure_ai" + ) + is True + ) # azure_ai should match azure models - assert _check_provider_match( - model_info={"litellm_provider": "azure"}, - custom_llm_provider="azure_ai" - ) is True + assert ( + _check_provider_match( + model_info={"litellm_provider": "azure"}, custom_llm_provider="azure_ai" + ) + is True + ) # azure_ai should NOT match other providers - assert _check_provider_match( - model_info={"litellm_provider": "anthropic"}, - custom_llm_provider="azure_ai" - ) is False + assert ( + _check_provider_match( + model_info={"litellm_provider": "anthropic"}, custom_llm_provider="azure_ai" + ) + is False + ) def test_check_provider_match_github_allows_upstream_provider_metadata(): @@ -61,20 +67,29 @@ def test_check_provider_match_github_allows_upstream_provider_metadata(): Test that github provider can match upstream provider metadata. GitHub Models can provide models from multiple providers. """ - assert _check_provider_match( - model_info={"litellm_provider": "openai"}, - custom_llm_provider="github", - ) is True + assert ( + _check_provider_match( + model_info={"litellm_provider": "openai"}, + custom_llm_provider="github", + ) + is True + ) - assert _check_provider_match( - model_info={"litellm_provider": "github"}, - custom_llm_provider="github", - ) is True + assert ( + _check_provider_match( + model_info={"litellm_provider": "github"}, + custom_llm_provider="github", + ) + is True + ) - assert _check_provider_match( - model_info={"litellm_provider": "anthropic"}, - custom_llm_provider="github", - ) is True + assert ( + _check_provider_match( + model_info={"litellm_provider": "anthropic"}, + custom_llm_provider="github", + ) + is True + ) def test_supports_function_calling_github_openai_alias(): @@ -604,7 +619,10 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_read_input_token_cost": {"type": "number"}, "cache_read_input_token_cost_above_200k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_272k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, + "cache_read_input_token_cost_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": { + "type": "number" + }, "cache_read_input_audio_token_cost": {"type": "number"}, "cache_read_input_token_cost_per_audio_token": {"type": "number"}, "cache_read_input_image_token_cost": {"type": "number"}, @@ -623,8 +641,12 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_token_above_272k_tokens": {"type": "number"}, "cache_read_input_token_cost_flex": {"type": "number"}, "cache_read_input_token_cost_priority": {"type": "number"}, - "cache_read_input_token_cost_above_200k_tokens_priority": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens_priority": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens_priority": { + "type": "number" + }, + "cache_read_input_token_cost_above_272k_tokens_priority": { + "type": "number" + }, "input_cost_per_token_flex": {"type": "number"}, "input_cost_per_token_priority": {"type": "number"}, "input_cost_per_token_above_200k_tokens_priority": {"type": "number"}, @@ -743,6 +765,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_multimodal": {"type": "boolean"}, "uses_embed_content": {"type": "boolean"}, "supports_reasoning": {"type": "boolean"}, + "supports_minimal_reasoning_effort": {"type": "boolean"}, "supports_none_reasoning_effort": {"type": "boolean"}, "supports_xhigh_reasoning_effort": {"type": "boolean"}, "supports_service_tier": {"type": "boolean"}, @@ -839,7 +862,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): }, } - prod_json = os.path.join(os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json") + prod_json = os.path.join( + os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json" + ) with open(prod_json, "r") as model_prices_file: actual_json = json.load(model_prices_file) assert isinstance(actual_json, dict) @@ -880,8 +905,10 @@ def test_max_tokens_consistency(): from pathlib import Path # Load the model configuration - config_path = Path(__file__).parent.parent.parent / "model_prices_and_context_window.json" - with open(config_path, 'r') as f: + config_path = ( + Path(__file__).parent.parent.parent / "model_prices_and_context_window.json" + ) + with open(config_path, "r") as f: models = json.load(f) inconsistencies = [] @@ -893,17 +920,19 @@ def test_max_tokens_consistency(): # Check if both max_tokens and max_output_tokens exist if isinstance(config, dict): - max_tokens = config.get('max_tokens') - max_output_tokens = config.get('max_output_tokens') + max_tokens = config.get("max_tokens") + max_output_tokens = config.get("max_output_tokens") # Only validate if both exist if max_tokens is not None and max_output_tokens is not None: if max_tokens != max_output_tokens: - inconsistencies.append({ - 'model': model_name, - 'max_tokens': max_tokens, - 'max_output_tokens': max_output_tokens - }) + inconsistencies.append( + { + "model": model_name, + "max_tokens": max_tokens, + "max_output_tokens": max_output_tokens, + } + ) if inconsistencies: error_msg = f"\n\n❌ Found {len(inconsistencies)} models with max_tokens != max_output_tokens:\n\n" @@ -2381,13 +2410,14 @@ def test_register_model_with_scientific_notation(): # Use a truly unique model name with uuid to avoid conflicts when tests run in parallel test_model_name = f"test-scientific-notation-model-{uuid.uuid4().hex[:12]}" - + # Clear LRU caches that might have stale data from litellm.utils import ( _invalidate_model_cost_lowercase_map, ) + _invalidate_model_cost_lowercase_map() - + model_cost_dict = { test_model_name: { "max_tokens": 8192, @@ -2406,7 +2436,7 @@ def test_register_model_with_scientific_notation(): assert registered_model["output_cost_per_token"] == 6e-07 assert registered_model["litellm_provider"] == "openai" assert registered_model["mode"] == "chat" - + # Clean up after test if test_model_name in litellm.model_cost: del litellm.model_cost[test_model_name] @@ -2734,7 +2764,9 @@ def test_model_info_for_openrouter_kimi_k2_5(): model_cost = json.load(f) model_info = model_cost.get("openrouter/moonshotai/kimi-k2.5") - assert model_info is not None, "Model not found in model_prices_and_context_window.json" + assert ( + model_info is not None + ), "Model not found in model_prices_and_context_window.json" assert model_info["litellm_provider"] == "openrouter" assert model_info["mode"] == "chat" @@ -2778,7 +2810,9 @@ def test_model_info_for_fireworks_short_form_models(): "fireworks_ai/accounts/fireworks/models/glm-4p7", ]: info = model_cost.get(key) - assert info is not None, f"{key} not found in model_prices_and_context_window.json" + assert ( + info is not None + ), f"{key} not found in model_prices_and_context_window.json" assert info["litellm_provider"] == "fireworks_ai" assert info["mode"] == "chat" assert info["input_cost_per_token"] == 6e-07 @@ -2792,7 +2826,9 @@ def test_model_info_for_fireworks_short_form_models(): "fireworks_ai/accounts/fireworks/models/minimax-m2p1", ]: info = model_cost.get(key) - assert info is not None, f"{key} not found in model_prices_and_context_window.json" + assert ( + info is not None + ), f"{key} not found in model_prices_and_context_window.json" assert info["litellm_provider"] == "fireworks_ai" assert info["mode"] == "chat" assert info["input_cost_per_token"] == 3e-07 @@ -2801,7 +2837,9 @@ def test_model_info_for_fireworks_short_form_models(): # kimi-k2p5: short-form only (long-form already existed) info = model_cost.get("fireworks_ai/kimi-k2p5") - assert info is not None, "fireworks_ai/kimi-k2p5 not found in model_prices_and_context_window.json" + assert ( + info is not None + ), "fireworks_ai/kimi-k2p5 not found in model_prices_and_context_window.json" assert info["litellm_provider"] == "fireworks_ai" assert info["mode"] == "chat" assert info["input_cost_per_token"] == 6e-07 @@ -3047,7 +3085,9 @@ class TestProxyLoggingBudgetAlerts: user_info = MagicMock() # Should not raise an error - await proxy_logging.budget_alerts(type="organization_budget", user_info=user_info) + await proxy_logging.budget_alerts( + type="organization_budget", user_info=user_info + ) async def test_budget_alerts_with_both_slack_and_email(self): """Test that budget_alerts calls both slack and email instances when both are in alerting.""" @@ -3103,11 +3143,13 @@ class TestProxyLoggingBudgetAlerts: type=alert_type, user_info=user_info ) - async def test_budget_alerts_soft_budget_with_alert_emails_bypasses_alerting_none(self): + async def test_budget_alerts_soft_budget_with_alert_emails_bypasses_alerting_none( + self, + ): """ Test that soft_budget alerts with alert_emails bypass the alerting=None check and send emails even when alerting is None. - + This tests the new logic that allows team-specific soft budget email alerts via metadata.soft_budget_alerting_emails to work even when global alerting is disabled. """ @@ -3143,7 +3185,9 @@ class TestProxyLoggingBudgetAlerts: type="soft_budget", user_info=user_info ) - async def test_budget_alerts_soft_budget_without_alert_emails_respects_alerting_none(self): + async def test_budget_alerts_soft_budget_without_alert_emails_respects_alerting_none( + self, + ): """ Test that soft_budget alerts WITHOUT alert_emails still respect alerting=None and do not send emails when alerting is None. @@ -3176,7 +3220,9 @@ class TestProxyLoggingBudgetAlerts: proxy_logging.slack_alerting_instance.budget_alerts.assert_not_called() proxy_logging.email_logging_instance.budget_alerts.assert_not_called() - async def test_budget_alerts_soft_budget_with_empty_alert_emails_respects_alerting_none(self): + async def test_budget_alerts_soft_budget_with_empty_alert_emails_respects_alerting_none( + self, + ): """ Test that soft_budget alerts with empty alert_emails list still respect alerting=None. """ @@ -3317,7 +3363,10 @@ def test_last_assistant_with_tool_calls_has_no_thinking_blocks_issue_18926(): {"type": "thinking", "thinking": "Let me analyze the requirements..."} ], "tool_calls": [ - {"id": "toolu_1", "function": {"name": "file_editor", "arguments": "{}"}} + { + "id": "toolu_1", + "function": {"name": "file_editor", "arguments": "{}"}, + } ], }, { @@ -3330,7 +3379,10 @@ def test_last_assistant_with_tool_calls_has_no_thinking_blocks_issue_18926(): # NO thinking_blocks - Claude sometimes doesn't include them "content": [{"type": "text", "text": "Let me explore more..."}], "tool_calls": [ - {"id": "toolu_2", "function": {"name": "file_editor", "arguments": "{}"}} + { + "id": "toolu_2", + "function": {"name": "file_editor", "arguments": "{}"}, + } ], }, ] @@ -3343,10 +3395,9 @@ def test_last_assistant_with_tool_calls_has_no_thinking_blocks_issue_18926(): # So we should NOT drop thinking - the combination tells us thinking is in use # The fix uses both checks: only drop if last has none AND no message has any - should_drop_thinking = ( - last_assistant_with_tool_calls_has_no_thinking_blocks(messages) - and not any_assistant_message_has_thinking_blocks(messages) - ) + should_drop_thinking = last_assistant_with_tool_calls_has_no_thinking_blocks( + messages + ) and not any_assistant_message_has_thinking_blocks(messages) assert should_drop_thinking is False @@ -3558,34 +3609,67 @@ class TestGetOptionalParamsDeepSeek: class TestIsStreamingRequest: def test_stream_true_in_kwargs(self): - assert _is_streaming_request(kwargs={"stream": True}, call_type="acompletion") is True + assert ( + _is_streaming_request(kwargs={"stream": True}, call_type="acompletion") + is True + ) def test_stream_false_in_kwargs(self): - assert _is_streaming_request(kwargs={"stream": False}, call_type="acompletion") is False + assert ( + _is_streaming_request(kwargs={"stream": False}, call_type="acompletion") + is False + ) def test_no_stream_in_kwargs(self): assert _is_streaming_request(kwargs={}, call_type="acompletion") is False def test_generate_content_stream_string(self): - assert _is_streaming_request(kwargs={}, call_type=CallTypes.generate_content_stream.value) is True + assert ( + _is_streaming_request( + kwargs={}, call_type=CallTypes.generate_content_stream.value + ) + is True + ) def test_agenerate_content_stream_string(self): - assert _is_streaming_request(kwargs={}, call_type=CallTypes.agenerate_content_stream.value) is True + assert ( + _is_streaming_request( + kwargs={}, call_type=CallTypes.agenerate_content_stream.value + ) + is True + ) def test_generate_content_stream_enum(self): - assert _is_streaming_request(kwargs={}, call_type=CallTypes.generate_content_stream) is True + assert ( + _is_streaming_request( + kwargs={}, call_type=CallTypes.generate_content_stream + ) + is True + ) def test_agenerate_content_stream_enum(self): - assert _is_streaming_request(kwargs={}, call_type=CallTypes.agenerate_content_stream) is True + assert ( + _is_streaming_request( + kwargs={}, call_type=CallTypes.agenerate_content_stream + ) + is True + ) def test_non_streaming_call_type_string(self): assert _is_streaming_request(kwargs={}, call_type="acompletion") is False def test_non_streaming_call_type_enum(self): - assert _is_streaming_request(kwargs={}, call_type=CallTypes.acompletion) is False + assert ( + _is_streaming_request(kwargs={}, call_type=CallTypes.acompletion) is False + ) def test_stream_true_overrides_non_streaming_call_type(self): - assert _is_streaming_request(kwargs={"stream": True}, call_type=CallTypes.acompletion) is True + assert ( + _is_streaming_request( + kwargs={"stream": True}, call_type=CallTypes.acompletion + ) + is True + ) class TestCallbackAsyncSyncSeparation: @@ -3679,37 +3763,27 @@ class TestMetadataNoneHandling: def test_metadata_none_get_previous_models(self): """kwargs.get("metadata") or {} should return {} when metadata is None.""" kwargs = {"metadata": None} - previous_models = (kwargs.get("metadata") or {}).get( - "previous_models", None - ) + previous_models = (kwargs.get("metadata") or {}).get("previous_models", None) assert previous_models is None def test_metadata_none_model_group_check(self): """'model_group' in (kwargs.get("metadata") or {}) should not raise TypeError.""" kwargs = {"metadata": None} - _is_litellm_router_call = "model_group" in ( - kwargs.get("metadata") or {} - ) + _is_litellm_router_call = "model_group" in (kwargs.get("metadata") or {}) assert _is_litellm_router_call is False def test_metadata_missing_key(self): """Should work when metadata key is completely absent.""" kwargs = {} - previous_models = (kwargs.get("metadata") or {}).get( - "previous_models", None - ) + previous_models = (kwargs.get("metadata") or {}).get("previous_models", None) assert previous_models is None def test_metadata_present_with_values(self): """Should work when metadata has actual values.""" kwargs = {"metadata": {"previous_models": ["model1"], "model_group": "test"}} - previous_models = (kwargs.get("metadata") or {}).get( - "previous_models", None - ) + previous_models = (kwargs.get("metadata") or {}).get("previous_models", None) assert previous_models == ["model1"] - _is_litellm_router_call = "model_group" in ( - kwargs.get("metadata") or {} - ) + _is_litellm_router_call = "model_group" in (kwargs.get("metadata") or {}) assert _is_litellm_router_call is True def test_metadata_none_causes_error_with_old_pattern(self): From 7c168ab173e67c03ad9128adf8343370f1b2d90d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Mar 2026 23:34:40 +0530 Subject: [PATCH 79/83] Fix gpt-5.4 using remote model cost map for tests --- .../llms/openai/test_gpt5_transformation.py | 103 +++++++++++++----- 1 file changed, 76 insertions(+), 27 deletions(-) diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 14e392a0998..aebab33e808 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -1,6 +1,7 @@ import pytest import litellm +from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai.openai import OpenAIConfig from litellm.utils import _is_explicitly_disabled_factory @@ -16,15 +17,23 @@ def gpt5_config() -> OpenAIGPT5Config: return OpenAIGPT5Config() +@pytest.fixture(autouse=True) +def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr( + litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url) + ) + litellm.add_known_models(model_cost_map=litellm.model_cost) + + def test_gpt5_supports_reasoning_effort(config: OpenAIConfig): assert "reasoning_effort" in config.get_supported_openai_params(model="gpt-5") assert "reasoning_effort" in config.get_supported_openai_params(model="gpt-5-mini") def test_gpt5_chat_does_not_support_reasoning_effort(config: OpenAIConfig): - assert ( - "reasoning_effort" - not in config.get_supported_openai_params(model="gpt-5-chat-latest") + assert "reasoning_effort" not in config.get_supported_openai_params( + model="gpt-5-chat-latest" ) @@ -133,7 +142,6 @@ def test_gpt5_codex_temperature_error(config: OpenAIConfig): ) - def test_gpt5_codex_temperature_one_allowed(config: OpenAIConfig): """Test that GPT-5-Codex allows temperature=1.""" params = config.map_openai_params( @@ -199,6 +207,8 @@ def test_gpt5_verbosity_parameter(config: OpenAIConfig): drop_params=False, ) assert params["verbosity"] == "low" + + def test_gpt5_1_reasoning_effort_none(config: OpenAIConfig): """Test that GPT-5.1 supports reasoning_effort='none' parameter. @@ -271,7 +281,9 @@ def test_gpt5_1_model_detection(gpt5_config: OpenAIGPT5Config): # codex/pro/chat variants do not support none assert not gpt5_config._supports_reasoning_effort_level("gpt-5.1-codex", "none") assert not gpt5_config._supports_reasoning_effort_level("gpt-5.1-codex-max", "none") - assert not gpt5_config._supports_reasoning_effort_level("gpt-5.2-chat-latest", "none") + assert not gpt5_config._supports_reasoning_effort_level( + "gpt-5.2-chat-latest", "none" + ) assert not gpt5_config._supports_reasoning_effort_level("gpt-5.2-pro", "none") assert not gpt5_config._supports_reasoning_effort_level("gpt-5", "none") assert not gpt5_config._supports_reasoning_effort_level("gpt-5-mini", "none") @@ -346,6 +358,7 @@ def test_gpt5_4_nano_allows_reasoning_effort_xhigh(config: OpenAIConfig): ) assert params["reasoning_effort"] == "xhigh" + def test_gpt5_4_nano_allows_reasoning_effort_none(config: OpenAIConfig): """gpt-5.4-nano supports reasoning_effort='none'.""" params = config.map_openai_params( @@ -356,6 +369,7 @@ def test_gpt5_4_nano_allows_reasoning_effort_none(config: OpenAIConfig): ) assert params["reasoning_effort"] == "none" + def test_gpt5_4_mini_allows_reasoning_effort_none(config: OpenAIConfig): """gpt-5.4-mini supports reasoning_effort='none'.""" params = config.map_openai_params( @@ -366,6 +380,7 @@ def test_gpt5_4_mini_allows_reasoning_effort_none(config: OpenAIConfig): ) assert params["reasoning_effort"] == "none" + def test_gpt5_4_allows_reasoning_effort_minimal(config: OpenAIConfig): """gpt-5.4 supports reasoning_effort='minimal'.""" params = config.map_openai_params( @@ -436,7 +451,9 @@ def test_gpt5_minimal_dict_triggers_validation(config: OpenAIConfig): """Dict with effort='minimal' triggers minimal model-support validation.""" with pytest.raises(litellm.utils.UnsupportedParamsError): config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "minimal", "summary": "detailed"}}, + non_default_params={ + "reasoning_effort": {"effort": "minimal", "summary": "detailed"} + }, optional_params={}, model="gpt-5.4-mini", drop_params=False, @@ -446,7 +463,9 @@ def test_gpt5_minimal_dict_triggers_validation(config: OpenAIConfig): def test_gpt5_minimal_dict_accepted_for_supported_model(config: OpenAIConfig): """Dict with effort='minimal' passes through for gpt-5.4+.""" params = config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "minimal", "summary": "detailed"}}, + non_default_params={ + "reasoning_effort": {"effort": "minimal", "summary": "detailed"} + }, optional_params={}, model="gpt-5.4", drop_params=False, @@ -520,7 +539,9 @@ def test_gpt5_unknown_model_passes_through_minimal(config: OpenAIConfig): def test_gpt5_normalizes_reasoning_effort_dict_with_summary(config: OpenAIConfig): """Dict with summary/generate_summary is normalized for chat completions.""" params = config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "high", "summary": "detailed"}}, + non_default_params={ + "reasoning_effort": {"effort": "high", "summary": "detailed"} + }, optional_params={}, model="gpt-5.4", drop_params=False, @@ -536,7 +557,9 @@ def test_gpt5_xhigh_dict_triggers_validation(config: OpenAIConfig): """ with pytest.raises(litellm.utils.UnsupportedParamsError): config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, + non_default_params={ + "reasoning_effort": {"effort": "xhigh", "summary": "detailed"} + }, optional_params={}, model="gpt-5.1", drop_params=False, @@ -546,7 +569,9 @@ def test_gpt5_xhigh_dict_triggers_validation(config: OpenAIConfig): def test_gpt5_xhigh_dict_accepted_for_supported_model(config: OpenAIConfig): """Dict with effort='xhigh' passes through for gpt-5.4+.""" params = config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, + non_default_params={ + "reasoning_effort": {"effort": "xhigh", "summary": "detailed"} + }, optional_params={}, model="gpt-5.4", drop_params=False, @@ -562,7 +587,10 @@ def test_gpt5_none_dict_with_tools_no_tool_drop(config: OpenAIConfig): """ tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] params = config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "none", "summary": "detailed"}, "tools": tools}, + non_default_params={ + "reasoning_effort": {"effort": "none", "summary": "detailed"}, + "tools": tools, + }, optional_params={}, model="gpt-5.4", drop_params=False, @@ -592,11 +620,15 @@ def test_gpt5_none_dict_with_sampling_params_allowed(config: OpenAIConfig): assert params["top_p"] == 0.9 -def test_gpt5_normalizes_reasoning_effort_dict_with_summary_from_optional_params(config: OpenAIConfig): +def test_gpt5_normalizes_reasoning_effort_dict_with_summary_from_optional_params( + config: OpenAIConfig, +): """reasoning_effort dict with summary in optional_params is normalized.""" params = config.map_openai_params( non_default_params={}, - optional_params={"reasoning_effort": {"effort": "medium", "summary": "detailed"}}, + optional_params={ + "reasoning_effort": {"effort": "medium", "summary": "detailed"} + }, model="gpt-5.4", drop_params=False, ) @@ -669,7 +701,7 @@ def test_gpt5_4_pro_rejects_non_default_temperature(config: OpenAIConfig): def test_gpt5_1_temperature_without_reasoning_effort(config: OpenAIConfig): """Test that GPT-5.1 supports any temperature when reasoning_effort is not specified. - + When reasoning_effort is not provided, it defaults to "none" for gpt-5.1, so temperature should be allowed. """ @@ -695,7 +727,7 @@ def test_gpt5_1_temperature_with_reasoning_effort_other_values(config: OpenAICon model="gpt-5.1", drop_params=False, ) - + # Test that temperature=1 is allowed with other reasoning_effort values for effort in ["low", "medium", "high"]: params = config.map_openai_params( @@ -708,7 +740,9 @@ def test_gpt5_1_temperature_with_reasoning_effort_other_values(config: OpenAICon assert params["reasoning_effort"] == effort -def test_gpt5_1_temperature_with_reasoning_effort_in_optional_params(config: OpenAIConfig): +def test_gpt5_1_temperature_with_reasoning_effort_in_optional_params( + config: OpenAIConfig, +): """Test that reasoning_effort can be in optional_params and still work correctly.""" # Test with reasoning_effort="none" in optional_params params = config.map_openai_params( @@ -718,7 +752,7 @@ def test_gpt5_1_temperature_with_reasoning_effort_in_optional_params(config: Ope drop_params=False, ) assert params["temperature"] == 0.5 - + # Test with reasoning_effort="low" in optional_params (should only allow temp=1) with pytest.raises(litellm.utils.UnsupportedParamsError): config.map_openai_params( @@ -728,6 +762,7 @@ def test_gpt5_1_temperature_with_reasoning_effort_in_optional_params(config: Ope drop_params=False, ) + def test_gpt5_1_temperature_drop_when_not_none(config: OpenAIConfig): """Test that GPT-5.1 drops temperature when reasoning_effort != 'none' and drop_params=True.""" params = config.map_openai_params( @@ -750,7 +785,7 @@ def test_gpt5_temperature_still_restricted(config: OpenAIConfig): model="gpt-5", drop_params=False, ) - + # temperature=1 should still work for gpt-5 params = config.map_openai_params( non_default_params={"temperature": 1.0}, @@ -843,7 +878,9 @@ def test_gpt5_search_supported_params(gpt5_config: OpenAIGPT5Config): "reasoning_effort", ] for param in rejected: - assert param not in supported, f"{param} should not be supported for search models" + assert ( + param not in supported + ), f"{param} should not be supported for search models" def test_gpt5_search_has_expected_params(gpt5_config: OpenAIGPT5Config): @@ -881,7 +918,11 @@ def test_gpt5_search_maps_max_tokens(config: OpenAIConfig): def test_gpt5_search_drops_unsupported_params(config: OpenAIConfig): """Test that search models drop unsupported params via map_openai_params.""" params = config.map_openai_params( - non_default_params={"n": 2, "temperature": 0.7, "tools": [{"type": "function"}]}, + non_default_params={ + "n": 2, + "temperature": 0.7, + "tools": [{"type": "function"}], + }, optional_params={}, model="gpt-5-search-api", drop_params=True, @@ -889,6 +930,8 @@ def test_gpt5_search_drops_unsupported_params(config: OpenAIConfig): assert "n" not in params assert "temperature" not in params assert "tools" not in params + + # GPT-5 unsupported params audit (validated via direct API calls) def test_gpt5_rejects_params_unsupported_by_openai(config: OpenAIConfig): """Params that OpenAI rejects for all GPT-5 reasoning models.""" @@ -902,9 +945,9 @@ def test_gpt5_rejects_params_unsupported_by_openai(config: OpenAIConfig): for model in ["gpt-5", "gpt-5-mini", "gpt-5-codex", "gpt-5.1", "gpt-5.2"]: supported = config.get_supported_openai_params(model=model) for param in rejected_params: - assert param not in supported, ( - f"{param} should not be supported for {model}" - ) + assert ( + param not in supported + ), f"{param} should not be supported for {model}" def test_gpt5_1_supports_logprobs_top_p(config: OpenAIConfig): @@ -913,16 +956,22 @@ def test_gpt5_1_supports_logprobs_top_p(config: OpenAIConfig): supported = config.get_supported_openai_params(model=model) assert "logprobs" in supported, f"logprobs should be supported for {model}" assert "top_p" in supported, f"top_p should be supported for {model}" - assert "top_logprobs" in supported, f"top_logprobs should be supported for {model}" + assert ( + "top_logprobs" in supported + ), f"top_logprobs should be supported for {model}" def test_gpt5_base_does_not_support_logprobs_top_p(config: OpenAIConfig): """Base gpt-5/gpt-5-mini do NOT support logprobs, top_p, top_logprobs.""" for model in ["gpt-5", "gpt-5-mini", "gpt-5-codex"]: supported = config.get_supported_openai_params(model=model) - assert "logprobs" not in supported, f"logprobs should not be supported for {model}" + assert ( + "logprobs" not in supported + ), f"logprobs should not be supported for {model}" assert "top_p" not in supported, f"top_p should not be supported for {model}" - assert "top_logprobs" not in supported, f"top_logprobs should not be supported for {model}" + assert ( + "top_logprobs" not in supported + ), f"top_logprobs should not be supported for {model}" def test_gpt5_1_logprobs_passthrough(config: OpenAIConfig): @@ -981,4 +1030,4 @@ def test_gpt5_1_logprobs_dropped_with_reasoning_effort(config: OpenAIConfig): ) assert "logprobs" not in params assert "top_p" not in params - assert params["reasoning_effort"] == "high" \ No newline at end of file + assert params["reasoning_effort"] == "high" From 8cb3a0f1d25adb7dd22cd0af881474b24690319e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Mar 2026 23:42:07 +0530 Subject: [PATCH 80/83] Fix ruff PLR0915 error --- .../responses/file_search/emulated_handler.py | 30 +++++---- litellm/responses/main.py | 66 +++++++++++-------- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 4700d673998..74a7d443c6c 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -409,6 +409,24 @@ async def _call_aresponses( return await aresponses(input=input, model=model, tools=tools, **kwargs) +def _prepare_emulated_file_search_call( + kwargs: Dict[str, Any], +) -> Tuple[bool, Dict[str, Any]]: + include_items: List[str] = list(kwargs.get("include") or []) + include_search_results = "file_search_call.results" in include_items + + original_stream = kwargs.get("stream") + updated_kwargs = kwargs + if original_stream: + verbose_logger.debug( + "Streaming is not yet supported for emulated file_search. " + "Disabling stream for this request." + ) + updated_kwargs = {**kwargs, "stream": False} + + return include_search_results, updated_kwargs + + async def aresponses_with_emulated_file_search( input: Any, model: str, @@ -423,17 +441,7 @@ async def aresponses_with_emulated_file_search( runs vector search, and synthesizes an OpenAI-format response. """ # Determine whether caller wants search_results populated in the output. - _include: List[str] = list(kwargs.get("include") or []) - _include_search_results = "file_search_call.results" in _include - - # Disable streaming for emulated file_search (not yet supported) - _original_stream = kwargs.get("stream") - if _original_stream: - verbose_logger.debug( - "Streaming is not yet supported for emulated file_search. " - "Disabling stream for this request." - ) - kwargs = {**kwargs, "stream": False} + _include_search_results, kwargs = _prepare_emulated_file_search_call(kwargs=kwargs) # 1. Replace file_search tools with function tool transformed_tools, all_vs_ids = _replace_file_search_tools(tools) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 16f97773fc6..c82574278ba 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -668,6 +668,43 @@ def _resolve_model_provider_for_responses( return model, custom_llm_provider +def _apply_managed_file_id_mapping( + input: Union[str, ResponseInputParam], + tools: Optional[Iterable[ToolParam]], + kwargs: Dict[str, Any], + local_vars: Dict[str, Any], +) -> tuple[Union[str, ResponseInputParam], Optional[Iterable[ToolParam]]]: + model_file_id_mapping = kwargs.get("model_file_id_mapping") + model_info_id = ( + kwargs.get("model_info", {}).get("id") + if isinstance(kwargs.get("model_info"), dict) + else None + ) + + input = cast( + Union[str, ResponseInputParam], + update_responses_input_with_model_file_ids( + input=input, + model_id=model_info_id, + model_file_id_mapping=model_file_id_mapping, + ), + ) + local_vars["input"] = input + + if tools: + tools = cast( + Optional[Iterable[ToolParam]], + update_responses_tools_with_model_file_ids( + tools=cast(Optional[List[Dict[str, Any]]], tools), + model_id=model_info_id, + model_file_id_mapping=model_file_id_mapping, + ), + ) + local_vars["tools"] = tools + + return input, tools + + @client def responses( input: Union[str, ResponseInputParam], @@ -764,35 +801,10 @@ def responses( ######################################################### # Update input and tools with provider-specific file IDs if managed files are used ######################################################### - model_file_id_mapping = kwargs.get("model_file_id_mapping") - model_info_id = ( - kwargs.get("model_info", {}).get("id") - if isinstance(kwargs.get("model_info"), dict) - else None + input, tools = _apply_managed_file_id_mapping( + input=input, tools=tools, kwargs=kwargs, local_vars=local_vars ) - input = cast( - Union[str, ResponseInputParam], - update_responses_input_with_model_file_ids( - input=input, - model_id=model_info_id, - model_file_id_mapping=model_file_id_mapping, - ), - ) - local_vars["input"] = input - - # Update tools with provider-specific file IDs if needed - if tools: - tools = cast( - Optional[Iterable[ToolParam]], - update_responses_tools_with_model_file_ids( - tools=cast(Optional[List[Dict[str, Any]]], tools), - model_id=model_info_id, - model_file_id_mapping=model_file_id_mapping, - ), - ) - local_vars["tools"] = tools - ######################################################### # Native MCP Responses API ######################################################### From 45f322caef5483acc3fbc63012e644bf55f7bd5b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Mar 2026 23:52:54 +0530 Subject: [PATCH 81/83] Fix ruff PLR0915 error --- litellm/proxy/common_request_processing.py | 215 ++++++++++++--------- 1 file changed, 123 insertions(+), 92 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 677ca7de3ce..9253b1e93b7 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -801,6 +801,108 @@ class ProxyBaseLLMRequestProcessing: json.dumps(self.data, indent=4, default=str), ) + async def _maybe_return_streaming_response( + self, + request: Request, + response: Any, + route_type: str, + user_api_key_dict: UserAPIKeyAuth, + proxy_logging_obj: ProxyLogging, + select_data_generator: Optional[Callable], + logging_obj: LiteLLMLoggingObj, + requested_model_from_client: Optional[str], + model_id: str, + cache_key: str, + api_base: str, + version: Optional[str], + response_cost: Any, + fastest_response_batch_completion: Any, + additional_headers: dict, + hidden_params: dict, + is_streaming_request: Optional[bool], + ) -> Optional[Any]: + if not ( + self._is_streaming_request( + data=self.data, is_streaming_request=is_streaming_request + ) + or self._is_streaming_response(response) + ): + return None + + custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + call_id=logging_obj.litellm_call_id, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + response_cost=response_cost, + model_region=getattr(user_api_key_dict, "allowed_model_region", ""), + fastest_response_batch_completion=fastest_response_batch_completion, + request_data=self.data, + hidden_params=hidden_params, + litellm_logging_obj=logging_obj, + **additional_headers, + ) + + callback_headers = await proxy_logging_obj.post_call_response_headers_hook( + data=self.data, + user_api_key_dict=user_api_key_dict, + response=response, + request_headers=dict(request.headers), + ) + if callback_headers: + custom_headers.update(callback_headers) + + if requested_model_from_client: + self.data["_litellm_client_requested_model"] = requested_model_from_client + + if route_type == "allm_passthrough_route": + if self._is_streaming_response(response): + if asyncio.iscoroutine(response): + generator = await response + else: + generator = response + return StreamingResponse( + content=generator, + status_code=status.HTTP_200_OK, + headers=custom_headers, + ) + return StreamingResponse( + content=response.aiter_bytes(), + status_code=response.status_code, + headers=custom_headers, + ) + elif route_type == "anthropic_messages": + if self._is_streaming_response(response): + selected_data_generator = ( + ProxyBaseLLMRequestProcessing.async_sse_data_generator( + response=response, + user_api_key_dict=user_api_key_dict, + request_data=self.data, + proxy_logging_obj=proxy_logging_obj, + ) + ) + return await create_response( + generator=selected_data_generator, + media_type="text/event-stream", + headers=custom_headers, + ) + return None + elif select_data_generator: + selected_data_generator = select_data_generator( + response=response, + user_api_key_dict=user_api_key_dict, + request_data=self.data, + ) + return await create_response( + generator=selected_data_generator, + media_type="text/event-stream", + headers=custom_headers, + ) + + return None + async def base_process_llm_request( self, request: Request, @@ -992,98 +1094,27 @@ class ProxyBaseLLMRequestProcessing: litellm_call_id=self.data.get("litellm_call_id", ""), status="success" ) ) - if self._is_streaming_request( - data=self.data, is_streaming_request=is_streaming_request - ) or self._is_streaming_response( - response - ): # use generate_responses to stream responses - custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( - user_api_key_dict=user_api_key_dict, - call_id=logging_obj.litellm_call_id, - model_id=model_id, - cache_key=cache_key, - api_base=api_base, - version=version, - response_cost=response_cost, - model_region=getattr(user_api_key_dict, "allowed_model_region", ""), - fastest_response_batch_completion=fastest_response_batch_completion, - request_data=self.data, - hidden_params=hidden_params, - litellm_logging_obj=logging_obj, - **additional_headers, - ) - - # Call response headers hook for streaming success - callback_headers = await proxy_logging_obj.post_call_response_headers_hook( - data=self.data, - user_api_key_dict=user_api_key_dict, - response=response, - request_headers=dict(request.headers), - ) - if callback_headers: - custom_headers.update(callback_headers) - - # Preserve the original client-requested model (pre-alias mapping) for downstream - # streaming generators. Pre-call processing can rewrite `self.data["model"]` for - # aliasing/routing, but the OpenAI-compatible response `model` field should reflect - # what the client sent. - if requested_model_from_client: - self.data[ - "_litellm_client_requested_model" - ] = requested_model_from_client - if route_type == "allm_passthrough_route": - # Check if response is an async generator - if self._is_streaming_response(response): - if asyncio.iscoroutine(response): - generator = await response - else: - generator = response - - # For passthrough routes, stream directly without error parsing - # since we're dealing with raw binary data (e.g., AWS event streams) - return StreamingResponse( - content=generator, - status_code=status.HTTP_200_OK, - headers=custom_headers, - ) - else: - # Traditional HTTP response with aiter_bytes - return StreamingResponse( - content=response.aiter_bytes(), - status_code=response.status_code, - headers=custom_headers, - ) - elif route_type == "anthropic_messages": - # Check if response is actually a streaming response (async generator) - # Non-streaming responses (dict) should be returned directly - # This handles cases like websearch_interception agentic loop - # which returns a non-streaming dict even for streaming requests - if self._is_streaming_response(response): - selected_data_generator = ( - ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=response, - user_api_key_dict=user_api_key_dict, - request_data=self.data, - proxy_logging_obj=proxy_logging_obj, - ) - ) - return await create_response( - generator=selected_data_generator, - media_type="text/event-stream", - headers=custom_headers, - ) - # Non-streaming response - fall through to normal response handling - elif select_data_generator: - selected_data_generator = select_data_generator( - response=response, - user_api_key_dict=user_api_key_dict, - request_data=self.data, - ) - return await create_response( - generator=selected_data_generator, - media_type="text/event-stream", - headers=custom_headers, - ) + streaming_response = await self._maybe_return_streaming_response( + request=request, + response=response, + route_type=route_type, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + select_data_generator=select_data_generator, + logging_obj=logging_obj, + requested_model_from_client=requested_model_from_client, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + response_cost=response_cost, + fastest_response_batch_completion=fastest_response_batch_completion, + additional_headers=additional_headers, + hidden_params=hidden_params, + is_streaming_request=is_streaming_request, + ) + if streaming_response is not None: + return streaming_response ### CALL HOOKS ### - modify outgoing data response = await proxy_logging_obj.post_call_success_hook( From ad8c321ea8e138dfc61f072bcfddea1dd7c8a6f3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Sun, 22 Mar 2026 00:35:35 +0530 Subject: [PATCH 82/83] fix(proxy): use union-attr mypy ignores for deferred logging hooks Made-with: Cursor --- litellm/proxy/common_request_processing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 15e37ad5358..d1aebe4dceb 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1091,7 +1091,7 @@ class ProxyBaseLLMRequestProcessing: cache_hit=cache_hit, ) - logging_obj._on_deferred_stream_complete = _on_deferred_stream_complete # type: ignore[attr-defined] + logging_obj._on_deferred_stream_complete = _on_deferred_stream_complete # type: ignore[union-attr] if route_type == "allm_passthrough_route": # Check if response is an async generator @@ -1153,7 +1153,7 @@ class ProxyBaseLLMRequestProcessing: # Clear the closure so guardrails run inline as before — this # preserves blocking behavior and avoids double invocation. if getattr(logging_obj, "_on_deferred_stream_complete", None): - logging_obj._on_deferred_stream_complete = None # type: ignore[attr-defined] + logging_obj._on_deferred_stream_complete = None # type: ignore[union-attr] response = await proxy_logging_obj.post_call_success_hook( data=self.data, user_api_key_dict=user_api_key_dict, @@ -1170,7 +1170,7 @@ class ProxyBaseLLMRequestProcessing: # returns before the deferred block), so _enqueue_fn is None — no-op. _enqueue_fn = getattr(logging_obj, "_enqueue_deferred_logging", None) if _enqueue_fn is not None: - logging_obj._enqueue_deferred_logging = None # type: ignore[attr-defined] + logging_obj._enqueue_deferred_logging = None # type: ignore[union-attr] try: _enqueue_fn() except Exception as e: @@ -1191,7 +1191,7 @@ class ProxyBaseLLMRequestProcessing: logging_obj, "_on_deferred_stream_complete", None ) if _deferred_fn is not None: - logging_obj._on_deferred_stream_complete = None # type: ignore[attr-defined] + logging_obj._on_deferred_stream_complete = None # type: ignore[union-attr] try: asyncio.create_task( logging_obj.async_success_handler( From c6bb7a5be5332b51214f85ad03950d7f1ca59f1d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 21 Mar 2026 15:05:53 -0700 Subject: [PATCH 83/83] Apply Black formatting to 11 files Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/litellm_core_utils/litellm_logging.py | 10 ++++++---- .../prompt_templates/common_utils.py | 9 +++++++-- .../adapters/transformation.py | 12 ++++++++--- litellm/llms/azure_ai/agents/handler.py | 12 +++-------- litellm/llms/moonshot/chat/transformation.py | 4 +++- .../llms/openai/chat/gpt_5_transformation.py | 4 +++- litellm/llms/ovhcloud/chat/transformation.py | 4 +--- .../llms/vertex_ai/gemini/transformation.py | 12 ++++++----- litellm/proxy/auth/user_api_key_auth.py | 12 +++++------ .../transformation.py | 4 +--- .../deployment_affinity_check.py | 20 ++++++++++--------- 11 files changed, 57 insertions(+), 46 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 56f7f305dca..5323f692b80 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1686,7 +1686,9 @@ class Logging(LiteLLMLoggingBaseClass): ) return logging_result - def _merge_hidden_params_from_response_into_metadata(self, logging_result: Any) -> None: + def _merge_hidden_params_from_response_into_metadata( + self, logging_result: Any + ) -> None: """ Copy response._hidden_params into litellm_params.metadata['hidden_params']. @@ -1704,9 +1706,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["litellm_params"].setdefault("metadata", {}) if self.model_call_details["litellm_params"]["metadata"] is None: self.model_call_details["litellm_params"]["metadata"] = {} - self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = getattr( - logging_result, "_hidden_params", {} - ) + self.model_call_details["litellm_params"]["metadata"][ + "hidden_params" + ] = getattr(logging_result, "_hidden_params", {}) def _process_hidden_params_and_response_cost( self, diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 2ef8e89c761..cec61405ebb 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -297,7 +297,10 @@ def _insert_user_continue_message( while i < len(result_messages): curr_message = result_messages[i] inserted_continue_message = False - if _counts_for_alternation(curr_message) and curr_message["role"] == "assistant": + if ( + _counts_for_alternation(curr_message) + and curr_message["role"] == "assistant" + ): # Preserve old behavior for malformed adjacent assistant sequences like # assistant(tool_calls) -> assistant(no-tool-calls) with no tool message. if i > 0 and result_messages[i - 1].get("role") == "assistant": @@ -584,7 +587,9 @@ def _decode_vector_store_ids_in_tools( continue parsed = parse_unified_id(vs_id) - provider_resource_id = parsed.get("provider_resource_id") if parsed else None + provider_resource_id = ( + parsed.get("provider_resource_id") if parsed else None + ) if not provider_resource_id: verbose_logger.warning( diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index b9a8d1de488..058d992e4a6 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -869,19 +869,25 @@ class LiteLLMAnthropicMessagesAdapter: # Handle array items if "items" in schema: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(schema["items"]) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( + schema["items"] + ) # Handle anyOf/oneOf/allOf for key in ("anyOf", "oneOf", "allOf"): if key in schema: for sub_schema in schema[key]: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(sub_schema) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( + sub_schema + ) # Handle $defs / definitions for key in ("$defs", "definitions"): if key in schema: for def_schema in schema[key].values(): - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(def_schema) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( + def_schema + ) def _add_system_message_to_messages( self, diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index 3fbda13d3c7..c3cd06ab4de 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -112,9 +112,7 @@ class AzureAIAgentsHandler: text_obj = content_item.get("text", {}) content = text_obj.get("value", "") raw_annotations = text_obj.get("annotations") - annotations = self._transform_annotations( - raw_annotations - ) + annotations = self._transform_annotations(raw_annotations) return content, annotations return "", None @@ -146,9 +144,7 @@ class AzureAIAgentsHandler: url_citation["start_index"] = ann["start_index"] if "end_index" in ann and "end_index" not in url_citation: url_citation["end_index"] = ann["end_index"] - result.append( - {"type": "url_citation", "url_citation": url_citation} - ) + result.append({"type": "url_citation", "url_citation": url_citation}) else: # Pass through unknown annotation types as-is result.append(ann) @@ -696,9 +692,7 @@ class AzureAIAgentsHandler: raw_annotations = content_item.get("text", {}).get( "annotations" ) - transformed = self._transform_annotations( - raw_annotations - ) + transformed = self._transform_annotations(raw_annotations) if transformed: if collected_annotations is None: collected_annotations = [] diff --git a/litellm/llms/moonshot/chat/transformation.py b/litellm/llms/moonshot/chat/transformation.py index c97bd6c4e12..e4d7b5f033b 100644 --- a/litellm/llms/moonshot/chat/transformation.py +++ b/litellm/llms/moonshot/chat/transformation.py @@ -168,7 +168,9 @@ class MoonshotChatConfig(OpenAIGPTConfig): if ( msg.get("role") == "assistant" and msg.get("tool_calls") - and not msg.get("reasoning_content") # Check using .get() which works for both dicts and Pydantic models + and not msg.get( + "reasoning_content" + ) # Check using .get() which works for both dicts and Pydantic models ): patched = dict(cast(dict, msg)) provider_fields = patched.get("provider_specific_fields") or {} diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 1a976beac81..fc48704cd10 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -235,7 +235,9 @@ class OpenAIGPT5Config(OpenAIGPTConfig): elif effective_effort == "minimal": # minimal is opt-out: unknown models pass through; only block when # the model map explicitly sets supports_minimal_reasoning_effort=false. - if self._is_reasoning_effort_level_explicitly_disabled(model, effective_effort): + if self._is_reasoning_effort_level_explicitly_disabled( + model, effective_effort + ): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) diff --git a/litellm/llms/ovhcloud/chat/transformation.py b/litellm/llms/ovhcloud/chat/transformation.py index 84090fafd31..1416b782f17 100644 --- a/litellm/llms/ovhcloud/chat/transformation.py +++ b/litellm/llms/ovhcloud/chat/transformation.py @@ -28,9 +28,7 @@ class OVHCloudChatConfig(OpenAIGPTConfig): """ supports_function_calling: Optional[bool] = None try: - model_info = _get_model_info_helper( - model, custom_llm_provider="ovhcloud" - ) + model_info = _get_model_info_helper(model, custom_llm_provider="ovhcloud") supports_function_calling = model_info.get( "supports_function_calling", None ) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index f6310778c71..7945c44d44c 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -555,7 +555,9 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 } } if "thought_signature" in invocation: - tc_part["thoughtSignature"] = invocation["thought_signature"] + tc_part["thoughtSignature"] = invocation[ + "thought_signature" + ] assistant_content.append(tc_part) # type: ignore # Re-inject toolResponse part if response is present @@ -566,11 +568,11 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 } if invocation.get("tool_type"): tr_dict["toolType"] = invocation["tool_type"] - tr_part: Dict[str, Any] = { - "toolResponse": tr_dict - } + tr_part: Dict[str, Any] = {"toolResponse": tr_dict} if "thought_signature" in invocation: - tr_part["thoughtSignature"] = invocation["thought_signature"] + tr_part["thoughtSignature"] = invocation[ + "thought_signature" + ] assistant_content.append(tr_part) # type: ignore msg_i += 1 diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index e2f06abc52f..800cca21dbe 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -920,9 +920,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 route=route, ) if _end_user_object is not None: - end_user_params["allowed_model_region"] = ( - _end_user_object.allowed_model_region - ) + end_user_params[ + "allowed_model_region" + ] = _end_user_object.allowed_model_region if _end_user_object.litellm_budget_table is not None: _apply_budget_limits_to_end_user_params( end_user_params=end_user_params, @@ -1499,9 +1499,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 if _end_user_object is not None: valid_token_dict.update(end_user_params) - valid_token_dict["end_user_object_permission"] = ( - _end_user_object.object_permission - ) + valid_token_dict[ + "end_user_object_permission" + ] = _end_user_object.object_permission # check if token is from litellm-ui, litellm ui makes keys to allow users to login with sso. These keys can only be used for LiteLLM UI functions # sso/login, ui/login, /key functions and /user functions diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 8da2a5e7c67..b6479a36998 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -232,9 +232,7 @@ class LiteLLMCompletionResponsesConfig: # only pass non-None values litellm_completion_request = { - k: v - for k, v in litellm_completion_request.items() - if v is not None + k: v for k, v in litellm_completion_request.items() if v is not None } return litellm_completion_request diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 728e689eaab..148b7fce0ee 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -70,9 +70,7 @@ class DeploymentAffinityCheck(CustomLogger): self.VALID_FLAGS, ) - def _get_effective_flags( - self, model_group: str - ) -> Tuple[bool, bool, bool]: + def _get_effective_flags(self, model_group: str) -> Tuple[bool, bool, bool]: """ Return (enable_user_key_affinity, enable_responses_api_affinity, enable_session_id_affinity) for the given model group. @@ -316,9 +314,11 @@ class DeploymentAffinityCheck(CustomLogger): request_kwargs = request_kwargs or {} typed_healthy_deployments = cast(List[dict], healthy_deployments) - enable_user_key, enable_responses_api, enable_session_id = ( - self._get_effective_flags(model) - ) + ( + enable_user_key, + enable_responses_api, + enable_session_id, + ) = self._get_effective_flags(model) # 1) Responses API continuity (high priority) if enable_responses_api: @@ -458,9 +458,11 @@ class DeploymentAffinityCheck(CustomLogger): return None # Resolve effective flags for this model group - enable_user_key, _enable_responses_api, enable_session_id = ( - self._get_effective_flags(deployment_model_name) - ) + ( + enable_user_key, + _enable_responses_api, + enable_session_id, + ) = self._get_effective_flags(deployment_model_name) if not enable_user_key and not enable_session_id: return None