From d98c71f07e808040d494002e9e4f9b5db8ce765c Mon Sep 17 00:00:00 2001 From: yogeshwaran10 Date: Sun, 11 Jan 2026 00:45:42 +0530 Subject: [PATCH 01/16] Fixes #18896 : Handle missing completion_tokens_details when reasoning_effort is not used --- .../vertex_and_google_ai_studio_gemini.py | 144 +++++++++++------- .../gemini/test_gemini_token_usage.py | 70 +++++++++ 2 files changed, 155 insertions(+), 59 deletions(-) create mode 100644 tests/test_litellm/llms/vertex_ai/gemini/test_gemini_token_usage.py diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 91100cf7d7b..e19bdc9edbd 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -310,9 +310,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): """ return Tools(googleSearch={}) - def _transform_computer_use_config( - self, computer_use_config: dict - ) -> dict: + def _transform_computer_use_config(self, computer_use_config: dict) -> dict: """ Transform Computer Use configuration to Gemini API format. @@ -323,7 +321,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Transformed computer use configuration for Gemini API """ transformed_config = {} - + # Transform environment values if needed if "environment" in computer_use_config: env_value = computer_use_config["environment"] @@ -339,13 +337,17 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): f"Invalid environment value for computer_use: {env_value}. " f"Supported: 'browser', 'unspecified', 'ENVIRONMENT_BROWSER', 'ENVIRONMENT_UNSPECIFIED'" ) - + # Transform excluded_predefined_functions to camelCase if "excluded_predefined_functions" in computer_use_config: - transformed_config["excludedPredefinedFunctions"] = computer_use_config["excluded_predefined_functions"] + transformed_config["excludedPredefinedFunctions"] = computer_use_config[ + "excluded_predefined_functions" + ] elif "excludedPredefinedFunctions" in computer_use_config: - transformed_config["excludedPredefinedFunctions"] = computer_use_config["excludedPredefinedFunctions"] - + transformed_config["excludedPredefinedFunctions"] = computer_use_config[ + "excludedPredefinedFunctions" + ] + return transformed_config def _extract_google_maps_retrieval_config( @@ -446,9 +448,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): value = _remove_strict_from_schema(value) for tool in value: - openai_function_object: Optional[ - ChatCompletionToolParamFunctionChunk - ] = None + openai_function_object: Optional[ChatCompletionToolParamFunctionChunk] = ( + None + ) if "function" in tool: # tools list _openai_function_object = ChatCompletionToolParamFunctionChunk( # type: ignore **tool["function"] @@ -553,7 +555,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "Invalid tool={}. Use `litellm.set_verbose` or `litellm --detailed_debug` to see raw request." ) -# Build list of Tool objects - each Tool should contain exactly one type + # Build list of Tool objects - each Tool should contain exactly one type # per Vertex AI API spec: "A Tool object should contain exactly one type of Tool" _tools_list: List[Tools] = [] @@ -570,11 +572,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): _tools_list.append(search_tool) if googleSearchRetrieval is not None: retrieval_tool = Tools() - retrieval_tool[VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value] = googleSearchRetrieval + retrieval_tool[VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value] = ( + googleSearchRetrieval + ) _tools_list.append(retrieval_tool) if enterpriseWebSearch is not None: enterprise_tool = Tools() - enterprise_tool[VertexToolName.ENTERPRISE_WEB_SEARCH.value] = enterpriseWebSearch + enterprise_tool[VertexToolName.ENTERPRISE_WEB_SEARCH.value] = ( + enterpriseWebSearch + ) _tools_list.append(enterprise_tool) if code_execution is not None: code_tool = Tools() @@ -593,7 +599,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): computer_tool[VertexToolName.COMPUTER_USE.value] = computerUse _tools_list.append(computer_tool) - # Add retrieval config to toolConfig if googleMaps has location data if google_maps_retrieval_config is not None: if "toolConfig" not in optional_params: @@ -710,8 +715,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): GeminiThinkingConfig with thinkingLevel and includeThoughts """ # Check if this is gemini-3-flash which supports MINIMAL thinking level - is_gemini3flash= model and ( - "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower() + is_gemini3flash = model and ( + "gemini-3-flash-preview" in model.lower() + or "gemini-3-flash" in model.lower() ) if reasoning_effort == "minimal": if is_gemini3flash: @@ -799,7 +805,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): thinking_budget = thinking_param.get("budget_tokens") params: GeminiThinkingConfig = {} - + # For Gemini 3+ models, use thinkingLevel instead of thinkingBudget if model and VertexGeminiConfig._is_gemini_3_or_newer(model): if thinking_enabled: @@ -808,11 +814,21 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): else: params["includeThoughts"] = True if thinking_budget >= 10000: - is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower() - params["thinkingLevel"] = "minimal" if is_gemini3flash else "low" + is_gemini3flash = ( + "gemini-3-flash-preview" in model.lower() + or "gemini-3-flash" in model.lower() + ) + params["thinkingLevel"] = ( + "minimal" if is_gemini3flash else "low" + ) else: - is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower() - params["thinkingLevel"] = "minimal" if is_gemini3flash else "low" + is_gemini3flash = ( + "gemini-3-flash-preview" in model.lower() + or "gemini-3-flash" in model.lower() + ) + params["thinkingLevel"] = ( + "minimal" if is_gemini3flash else "low" + ) else: # Thinking disabled params["includeThoughts"] = False @@ -824,7 +840,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): params["includeThoughts"] = True if thinking_budget is not None and isinstance(thinking_budget, int): params["thinkingBudget"] = thinking_budget - + return params def map_response_modalities(self, value: list) -> list: @@ -980,16 +996,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): param_description="thinking_budget", ) if VertexGeminiConfig._is_gemini_3_or_newer(model): - optional_params[ - "thinkingConfig" - ] = VertexGeminiConfig._map_reasoning_effort_to_thinking_level( - value, model + optional_params["thinkingConfig"] = ( + VertexGeminiConfig._map_reasoning_effort_to_thinking_level( + value, model + ) ) else: - optional_params[ - "thinkingConfig" - ] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget( - value, model + optional_params["thinkingConfig"] = ( + VertexGeminiConfig._map_reasoning_effort_to_thinking_budget( + value, model + ) ) elif param == "thinking": # Validate no conflict with thinking_level @@ -998,11 +1014,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): param_name="thinking", param_description="thinking_budget", ) - optional_params[ - "thinkingConfig" - ] = VertexGeminiConfig._map_thinking_param( - cast(AnthropicThinkingParam, value), - model=model, + optional_params["thinkingConfig"] = ( + VertexGeminiConfig._map_thinking_param( + cast(AnthropicThinkingParam, value), + model=model, + ) ) elif param == "modalities" and isinstance(value, list): response_modalities = self.map_response_modalities(value) @@ -1036,8 +1052,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ): # For gemini-3-flash-preview, default to "minimal" to match Gemini 2.5 Flash behavior # For other Gemini 3 models, default to "low" - is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower() - thinking_config["thinkingLevel"] = "minimal" if is_gemini3flash else "low" + is_gemini3flash = ( + "gemini-3-flash-preview" in model.lower() + or "gemini-3-flash" in model.lower() + ) + thinking_config["thinkingLevel"] = ( + "minimal" if is_gemini3flash else "low" + ) optional_params["thinkingConfig"] = thinking_config return optional_params @@ -1226,7 +1247,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): block: ChatCompletionThinkingBlock = { "type": "thinking", "thinking": thinking_text, - } + } signature = part.get("thoughtSignature") if signature is not None: block["signature"] = signature @@ -1360,10 +1381,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): _tool_response_chunk["provider_specific_fields"] = { # type: ignore "thought_signature": thought_signature } - _tool_response_chunk[ - "id" - ] = _encode_tool_call_id_with_signature( - _tool_response_chunk["id"] or "", thought_signature + _tool_response_chunk["id"] = ( + _encode_tool_call_id_with_signature( + _tool_response_chunk["id"] or "", thought_signature + ) ) _tools.append(_tool_response_chunk) cumulative_tool_call_idx += 1 @@ -1551,13 +1572,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif modality == "IMAGE": response_tokens_details.image_tokens = token_count - # Calculate text_tokens if not explicitly provided in candidatesTokensDetails - # candidatesTokenCount includes all modalities, so: text = total - (image + audio) + # Calculate text_tokens if not explicitly provided in candidatesTokensDetails + # candidatesTokenCount includes all modalities, so: text = total - (image + audio) + candidates_token_count = usage_metadata.get("candidatesTokenCount", 0) + if candidates_token_count > 0: + if response_tokens_details is None: + response_tokens_details = CompletionTokensDetailsWrapper() if response_tokens_details.text_tokens is None: - candidates_token_count = usage_metadata.get("candidatesTokenCount", 0) image_tokens = response_tokens_details.image_tokens or 0 audio_tokens_candidate = response_tokens_details.audio_tokens or 0 - calculated_text_tokens = candidates_token_count - image_tokens - audio_tokens_candidate + calculated_text_tokens = ( + candidates_token_count - image_tokens - audio_tokens_candidate + ) response_tokens_details.text_tokens = calculated_text_tokens ######################################################### @@ -2076,28 +2102,28 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ## ADD METADATA TO RESPONSE ## setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) - model_response._hidden_params[ - "vertex_ai_grounding_metadata" - ] = grounding_metadata + model_response._hidden_params["vertex_ai_grounding_metadata"] = ( + grounding_metadata + ) setattr( model_response, "vertex_ai_url_context_metadata", url_context_metadata ) - model_response._hidden_params[ - "vertex_ai_url_context_metadata" - ] = url_context_metadata + model_response._hidden_params["vertex_ai_url_context_metadata"] = ( + url_context_metadata + ) setattr(model_response, "vertex_ai_safety_results", safety_ratings) - model_response._hidden_params[ - "vertex_ai_safety_results" - ] = safety_ratings # older approach - maintaining to prevent regressions + model_response._hidden_params["vertex_ai_safety_results"] = ( + safety_ratings # older approach - maintaining to prevent regressions + ) ## ADD CITATION METADATA ## setattr(model_response, "vertex_ai_citation_metadata", citation_metadata) - model_response._hidden_params[ - "vertex_ai_citation_metadata" - ] = citation_metadata # older approach - maintaining to prevent regressions + model_response._hidden_params["vertex_ai_citation_metadata"] = ( + citation_metadata # older approach - maintaining to prevent regressions + ) except Exception as e: raise VertexAIError( diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_gemini_token_usage.py b/tests/test_litellm/llms/vertex_ai/gemini/test_gemini_token_usage.py new file mode 100644 index 00000000000..c8894ca7d5c --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_gemini_token_usage.py @@ -0,0 +1,70 @@ + +import sys, os +import pytest +sys.path.insert(0, os.path.abspath('../../../../../')) + +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig +from litellm.types.llms.vertex_ai import UsageMetadata + +def test_gemini_3_flash_preview_token_usage_fallback(): + """Test fallback logic when candidatesTokensDetails is missing (e.g. Gemini 3 Flash Preview).""" + v = VertexGeminiConfig() + + usage_metadata_dict = { + "promptTokenCount": 2145, + "candidatesTokenCount": 509, + "totalTokenCount": 2654, + # candidatesTokensDetails intentionally omitted + } + + completion_response = {"usageMetadata": usage_metadata_dict} + result = v._calculate_usage(completion_response=completion_response) + + assert result.completion_tokens == 509 + assert result.prompt_tokens == 2145 + assert result.total_tokens == 2654 + + # Text tokens should be derived from candidatesTokenCount + assert result.completion_tokens_details is not None + assert result.completion_tokens_details.text_tokens == 509 + assert result.completion_tokens_details.image_tokens is None + assert result.completion_tokens_details.audio_tokens is None + +def test_gemini_no_reasoning_fallback(): + """Test fallback when reasoning_effort is absent and details are missing.""" + v = VertexGeminiConfig() + + usage_metadata_dict = { + "promptTokenCount": 100, + "candidatesTokenCount": 264, + "totalTokenCount": 364, + } + + completion_response = {"usageMetadata": usage_metadata_dict} + result = v._calculate_usage(completion_response=completion_response) + + assert result.completion_tokens == 264 + assert result.completion_tokens_details is not None + assert result.completion_tokens_details.text_tokens == 264 + assert result.completion_tokens_details.reasoning_tokens is None or result.completion_tokens_details.reasoning_tokens == 0 + +def test_gemini_token_usage_standard_response(): + """Verify that standard responses with details are computed correctly and not overwritten.""" + v = VertexGeminiConfig() + + usage_metadata_dict = { + "promptTokenCount": 100, + "candidatesTokenCount": 50, + "totalTokenCount": 150, + "candidatesTokensDetails": [ + {"modality": "TEXT", "tokenCount": 40}, + {"modality": "IMAGE", "tokenCount": 10} + ] + } + + completion_response = {"usageMetadata": usage_metadata_dict} + result = v._calculate_usage(completion_response=completion_response) + + assert result.completion_tokens == 50 + assert result.completion_tokens_details.text_tokens == 40 + assert result.completion_tokens_details.image_tokens == 10 From 0bc04ce129ed86781f332b4429a16eaa95a2327c Mon Sep 17 00:00:00 2001 From: Chesars Date: Sat, 10 Jan 2026 19:39:04 +0000 Subject: [PATCH 02/16] fix: correct pricing for openrouter/openai/gpt-oss-20b MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updated pricing from incorrect values to match OpenRouter's official rates: - Input: $0.18/M → $0.02/M tokens (1.8e-07 → 2e-08) - Output: $0.80/M → $0.10/M tokens (8e-07 → 1e-07) --- litellm/model_prices_and_context_window_backup.json | 4 ++-- model_prices_and_context_window.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 73579db75cd..89824ba13d4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23340,13 +23340,13 @@ "supports_tool_choice": true }, "openrouter/openai/gpt-oss-20b": { - "input_cost_per_token": 1.8e-07, + "input_cost_per_token": 2e-08, "litellm_provider": "openrouter", "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 8e-07, + "output_cost_per_token": 1e-07, "source": "https://openrouter.ai/openai/gpt-oss-20b", "supports_function_calling": true, "supports_parallel_function_calling": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 73579db75cd..89824ba13d4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23340,13 +23340,13 @@ "supports_tool_choice": true }, "openrouter/openai/gpt-oss-20b": { - "input_cost_per_token": 1.8e-07, + "input_cost_per_token": 2e-08, "litellm_provider": "openrouter", "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 8e-07, + "output_cost_per_token": 1e-07, "source": "https://openrouter.ai/openai/gpt-oss-20b", "supports_function_calling": true, "supports_parallel_function_calling": true, From a9e57cf2721596c2ba1f45d941d5d457f1e561a0 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 9 Jan 2026 12:30:36 +0530 Subject: [PATCH 03/16] [Bug]: Add Custom CA certificates to boto3 clients --- litellm/llms/bedrock/base_aws_llm.py | 44 ++- litellm/llms/bedrock/common_utils.py | 2 +- litellm/llms/bedrock/files/handler.py | 1 + .../llms/bedrock/test_bedrock_ssl_verify.py | 349 ++++++++++++++++++ 4 files changed, 392 insertions(+), 4 deletions(-) create mode 100644 tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index e9cea23ea4a..18e9deb53b0 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -74,6 +74,41 @@ class BaseAWSLLM: "aws_external_id", ] + def _get_ssl_verify(self): + """ + Get SSL verification setting for boto3 clients. + + This ensures that custom CA certificates are properly used for all AWS API calls, + including STS and Bedrock services. + + Returns: + Union[bool, str]: SSL verification setting - False to disable, True to enable, + or a string path to a CA bundle file + """ + import litellm + from litellm.secret_managers.main import str_to_bool + + # Check environment variable first (highest priority) + ssl_verify = os.getenv("SSL_VERIFY", litellm.ssl_verify) + + # Convert string "False"/"True" to boolean + if isinstance(ssl_verify, str): + # Check if it's a file path + if os.path.exists(ssl_verify): + return ssl_verify + # Otherwise try to convert to boolean + ssl_verify_bool = str_to_bool(ssl_verify) + if ssl_verify_bool is not None: + ssl_verify = ssl_verify_bool + + # Check SSL_CERT_FILE environment variable for custom CA bundle + if ssl_verify is True or ssl_verify == "True": + ssl_cert_file = os.getenv("SSL_CERT_FILE") + if ssl_cert_file and os.path.exists(ssl_cert_file): + return ssl_cert_file + + return ssl_verify + def get_cache_key(self, credential_args: Dict[str, Optional[str]]) -> str: """ Generate a unique cache key based on the credential arguments. @@ -569,6 +604,7 @@ class BaseAWSLLM: "sts", region_name=aws_region_name, endpoint_url=sts_endpoint, + verify=self._get_ssl_verify(), ) # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html @@ -625,7 +661,7 @@ class BaseAWSLLM: # Create an STS client without credentials with tracer.trace("boto3.client(sts) for manual IRSA"): - sts_client = boto3.client("sts", region_name=region) + sts_client = boto3.client("sts", region_name=region, verify=self._get_ssl_verify()) # Manually assume the IRSA role with the session name verbose_logger.debug( @@ -648,6 +684,7 @@ class BaseAWSLLM: aws_access_key_id=irsa_creds["AccessKeyId"], aws_secret_access_key=irsa_creds["SecretAccessKey"], aws_session_token=irsa_creds["SessionToken"], + verify=self._get_ssl_verify(), ) # Get current caller identity for debugging @@ -686,7 +723,7 @@ class BaseAWSLLM: verbose_logger.debug("Same account role assumption, using automatic IRSA") with tracer.trace("boto3.client(sts) with automatic IRSA"): - sts_client = boto3.client("sts", region_name=region) + sts_client = boto3.client("sts", region_name=region, verify=self._get_ssl_verify()) # Get current caller identity for debugging try: @@ -809,7 +846,7 @@ class BaseAWSLLM: # This allows the web identity token to work automatically if aws_access_key_id is None and aws_secret_access_key is None: with tracer.trace("boto3.client(sts)"): - sts_client = boto3.client("sts") + sts_client = boto3.client("sts", verify=self._get_ssl_verify()) else: with tracer.trace("boto3.client(sts)"): sts_client = boto3.client( @@ -817,6 +854,7 @@ class BaseAWSLLM: aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, + verify=self._get_ssl_verify(), ) assume_role_params = { diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index d62a8bae425..f4b5de8f7c0 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -260,7 +260,7 @@ def init_bedrock_client( status_code=401, ) - sts_client = boto3.client("sts") + sts_client = boto3.client("sts", verify=ssl_verify) # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html diff --git a/litellm/llms/bedrock/files/handler.py b/litellm/llms/bedrock/files/handler.py index d6177e090d5..0350271dc44 100644 --- a/litellm/llms/bedrock/files/handler.py +++ b/litellm/llms/bedrock/files/handler.py @@ -142,6 +142,7 @@ class BedrockFilesHandler(BaseAWSLLM): aws_secret_access_key=credentials.secret_key, aws_session_token=credentials.token, region_name=aws_region_name, + verify=self._get_ssl_verify(), ) # Download file from S3 diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py b/tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py new file mode 100644 index 00000000000..9142de295ea --- /dev/null +++ b/tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py @@ -0,0 +1,349 @@ +""" +Test SSL verification for AWS Bedrock boto3 clients. + +This test ensures that custom CA certificates are properly passed to all boto3 clients +(STS and Bedrock services) to support internal certificate authorities. + +Issue: https://github.com/BerriAI/litellm/issues/XXXX +User reported that SSL_CERT_FILE environment variable and ssl_verify config were not +being applied to boto3 clients, causing "certificate verify failed" errors. +""" + +import os +import sys +import tempfile +from unittest.mock import MagicMock, Mock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import init_bedrock_client + + +class TestBedrockSSLVerify: + """Test suite for SSL verification in Bedrock boto3 clients.""" + + def test_base_aws_llm_get_ssl_verify_default(self): + """Test that _get_ssl_verify returns default value when no custom config is set.""" + base_aws = BaseAWSLLM() + + # Clear any environment variables + os.environ.pop("SSL_VERIFY", None) + os.environ.pop("SSL_CERT_FILE", None) + + # Reset litellm.ssl_verify to default + litellm.ssl_verify = True + + ssl_verify = base_aws._get_ssl_verify() + assert ssl_verify is True + + def test_base_aws_llm_get_ssl_verify_false(self): + """Test that _get_ssl_verify returns False when SSL verification is disabled.""" + base_aws = BaseAWSLLM() + + # Set SSL_VERIFY to False via environment + os.environ["SSL_VERIFY"] = "False" + + ssl_verify = base_aws._get_ssl_verify() + assert ssl_verify is False + + # Clean up + os.environ.pop("SSL_VERIFY", None) + + def test_base_aws_llm_get_ssl_verify_custom_ca_bundle(self): + """Test that _get_ssl_verify returns custom CA bundle path when SSL_CERT_FILE is set.""" + base_aws = BaseAWSLLM() + + # Create a temporary CA bundle file + with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f: + f.write("-----BEGIN CERTIFICATE-----\n") + f.write("FAKE CERTIFICATE FOR TESTING\n") + f.write("-----END CERTIFICATE-----\n") + ca_bundle_path = f.name + + try: + # Set SSL_CERT_FILE environment variable + os.environ["SSL_CERT_FILE"] = ca_bundle_path + os.environ.pop("SSL_VERIFY", None) + litellm.ssl_verify = True + + ssl_verify = base_aws._get_ssl_verify() + assert ssl_verify == ca_bundle_path + finally: + # Clean up + os.environ.pop("SSL_CERT_FILE", None) + os.unlink(ca_bundle_path) + + def test_base_aws_llm_get_ssl_verify_litellm_config(self): + """Test that _get_ssl_verify uses litellm.ssl_verify when set.""" + base_aws = BaseAWSLLM() + + # Clear environment variables + os.environ.pop("SSL_VERIFY", None) + os.environ.pop("SSL_CERT_FILE", None) + + # Create a temporary CA bundle file + with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f: + f.write("-----BEGIN CERTIFICATE-----\n") + f.write("FAKE CERTIFICATE FOR TESTING\n") + f.write("-----END CERTIFICATE-----\n") + ca_bundle_path = f.name + + try: + # Set litellm.ssl_verify to custom CA bundle + litellm.ssl_verify = ca_bundle_path + + ssl_verify = base_aws._get_ssl_verify() + # When ssl_verify is a path, it should be returned directly + assert ssl_verify == ca_bundle_path + finally: + # Clean up + litellm.ssl_verify = True + os.unlink(ca_bundle_path) + + @patch("boto3.client") + def test_init_bedrock_client_passes_ssl_verify_to_sts(self, mock_boto3_client): + """Test that init_bedrock_client passes ssl_verify to STS client.""" + # Create a temporary CA bundle file + with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f: + f.write("-----BEGIN CERTIFICATE-----\n") + f.write("FAKE CERTIFICATE FOR TESTING\n") + f.write("-----END CERTIFICATE-----\n") + ca_bundle_path = f.name + + try: + # Set SSL_CERT_FILE environment variable + os.environ["SSL_CERT_FILE"] = ca_bundle_path + litellm.ssl_verify = True + + # Mock the STS client and Bedrock client + mock_sts_client = MagicMock() + mock_sts_response = { + "Credentials": { + "AccessKeyId": "test_access_key", + "SecretAccessKey": "test_secret_key", + "SessionToken": "test_session_token", + } + } + mock_sts_client.assume_role.return_value = mock_sts_response + + mock_bedrock_client = MagicMock() + + # Configure mock to return different clients based on service name + def side_effect(service_name=None, **kwargs): + if service_name == "sts": + return mock_sts_client + elif service_name == "bedrock-runtime": + return mock_bedrock_client + return MagicMock() + + mock_boto3_client.side_effect = side_effect + + # Call init_bedrock_client with role assumption + client = init_bedrock_client( + aws_region_name="us-west-2", + aws_access_key_id="test_key", + aws_secret_access_key="test_secret", + aws_role_name="arn:aws:iam::123456789012:role/test-role", + aws_session_name="test-session", + ) + + # Verify that boto3.client was called with verify parameter for STS + sts_calls = [ + call for call in mock_boto3_client.call_args_list + if (len(call[0]) > 0 and call[0][0] == "sts") or + ("service_name" not in call[1]) # STS calls don't use service_name kwarg + ] + + assert len(sts_calls) > 0, "STS client should have been created" + + # Check that verify parameter was passed to STS client + sts_call = sts_calls[0] + assert "verify" in sts_call[1], "verify parameter should be passed to STS client" + assert sts_call[1]["verify"] == ca_bundle_path, f"verify should be set to CA bundle path, got {sts_call[1]['verify']}" + + # Verify that boto3.client was called with verify parameter for Bedrock + bedrock_calls = [ + call for call in mock_boto3_client.call_args_list + if "service_name" in call[1] and call[1]["service_name"] == "bedrock-runtime" + ] + + assert len(bedrock_calls) > 0, "Bedrock client should have been created" + + bedrock_call = bedrock_calls[0] + assert "verify" in bedrock_call[1], "verify parameter should be passed to Bedrock client" + assert bedrock_call[1]["verify"] == ca_bundle_path, f"verify should be set to CA bundle path, got {bedrock_call[1]['verify']}" + + finally: + # Clean up + os.environ.pop("SSL_CERT_FILE", None) + os.unlink(ca_bundle_path) + + @patch("boto3.client") + def test_base_aws_llm_auth_with_role_passes_ssl_verify(self, mock_boto3_client): + """Test that _auth_with_aws_role passes ssl_verify to STS client.""" + base_aws = BaseAWSLLM() + + # Create a temporary CA bundle file + with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f: + f.write("-----BEGIN CERTIFICATE-----\n") + f.write("FAKE CERTIFICATE FOR TESTING\n") + f.write("-----END CERTIFICATE-----\n") + ca_bundle_path = f.name + + try: + # Set SSL_CERT_FILE environment variable + os.environ["SSL_CERT_FILE"] = ca_bundle_path + litellm.ssl_verify = True + + # Mock the STS client + mock_sts_client = MagicMock() + mock_sts_response = { + "Credentials": { + "AccessKeyId": "test_access_key", + "SecretAccessKey": "test_secret_key", + "SessionToken": "test_session_token", + "Expiration": "2025-01-10T00:00:00Z", + } + } + + # Convert Expiration to datetime + from datetime import datetime, timezone + mock_sts_response["Credentials"]["Expiration"] = datetime.now(timezone.utc) + + mock_sts_client.assume_role.return_value = mock_sts_response + mock_boto3_client.return_value = mock_sts_client + + # Call _auth_with_aws_role + credentials, ttl = base_aws._auth_with_aws_role( + aws_access_key_id="test_key", + aws_secret_access_key="test_secret", + aws_session_token=None, + aws_role_name="arn:aws:iam::123456789012:role/test-role", + aws_session_name="test-session", + ) + + # Verify that boto3.client was called with verify parameter + assert mock_boto3_client.called, "boto3.client should have been called" + + call_kwargs = mock_boto3_client.call_args[1] + assert "verify" in call_kwargs, "verify parameter should be passed to STS client" + assert call_kwargs["verify"] == ca_bundle_path, f"verify should be set to CA bundle path, got {call_kwargs['verify']}" + + finally: + # Clean up + os.environ.pop("SSL_CERT_FILE", None) + os.unlink(ca_bundle_path) + + @patch("litellm.llms.bedrock.base_aws_llm.get_secret") + @patch("boto3.client") + def test_base_aws_llm_auth_with_web_identity_passes_ssl_verify(self, mock_boto3_client, mock_get_secret): + """Test that _auth_with_web_identity_token passes ssl_verify to STS client.""" + base_aws = BaseAWSLLM() + + # Create a temporary CA bundle file + with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f: + f.write("-----BEGIN CERTIFICATE-----\n") + f.write("FAKE CERTIFICATE FOR TESTING\n") + f.write("-----END CERTIFICATE-----\n") + ca_bundle_path = f.name + + try: + # Set SSL_CERT_FILE environment variable + os.environ["SSL_CERT_FILE"] = ca_bundle_path + litellm.ssl_verify = True + + # Mock get_secret to return the token + mock_get_secret.return_value = "mocked_oidc_token" + + # Mock the STS client + mock_sts_client = MagicMock() + mock_sts_response = { + "Credentials": { + "AccessKeyId": "test_access_key", + "SecretAccessKey": "test_secret_key", + "SessionToken": "test_session_token", + }, + "PackedPolicySize": 100, + } + + mock_sts_client.assume_role_with_web_identity.return_value = mock_sts_response + + # Mock boto3.Session + mock_session = MagicMock() + mock_credentials = MagicMock() + mock_session.get_credentials.return_value = mock_credentials + + mock_boto3_client.return_value = mock_sts_client + + with patch("boto3.Session", return_value=mock_session): + # Call _auth_with_web_identity_token + credentials, ttl = base_aws._auth_with_web_identity_token( + aws_web_identity_token="test_token", + aws_role_name="arn:aws:iam::123456789012:role/test-role", + aws_session_name="test-session", + aws_region_name="us-west-2", + aws_sts_endpoint=None, + ) + + # Verify that boto3.client was called with verify parameter + assert mock_boto3_client.called, "boto3.client should have been called" + + call_kwargs = mock_boto3_client.call_args[1] + assert "verify" in call_kwargs, "verify parameter should be passed to STS client" + assert call_kwargs["verify"] == ca_bundle_path, f"verify should be set to CA bundle path, got {call_kwargs['verify']}" + + finally: + # Clean up + os.environ.pop("SSL_CERT_FILE", None) + os.unlink(ca_bundle_path) + + def test_ssl_verify_priority_env_over_litellm_config(self): + """Test that SSL_VERIFY environment variable takes priority over litellm.ssl_verify.""" + base_aws = BaseAWSLLM() + + # Set litellm.ssl_verify to True + litellm.ssl_verify = True + + # Set SSL_VERIFY environment variable to False + os.environ["SSL_VERIFY"] = "False" + + try: + ssl_verify = base_aws._get_ssl_verify() + assert ssl_verify is False, "Environment variable should take priority" + finally: + # Clean up + os.environ.pop("SSL_VERIFY", None) + litellm.ssl_verify = True + + def test_ssl_cert_file_priority_over_default(self): + """Test that SSL_CERT_FILE takes priority when ssl_verify is True.""" + base_aws = BaseAWSLLM() + + # Create a temporary CA bundle file + with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f: + f.write("-----BEGIN CERTIFICATE-----\n") + f.write("FAKE CERTIFICATE FOR TESTING\n") + f.write("-----END CERTIFICATE-----\n") + ca_bundle_path = f.name + + try: + # Set SSL_CERT_FILE environment variable + os.environ["SSL_CERT_FILE"] = ca_bundle_path + os.environ.pop("SSL_VERIFY", None) + litellm.ssl_verify = True + + ssl_verify = base_aws._get_ssl_verify() + assert ssl_verify == ca_bundle_path, "SSL_CERT_FILE should be used when ssl_verify is True" + finally: + # Clean up + os.environ.pop("SSL_CERT_FILE", None) + os.unlink(ca_bundle_path) + + +if __name__ == "__main__": + # Run tests + pytest.main([__file__, "-v", "-s"]) From aa97a34a833da4ac4a379252844a5cadce170b30 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 12 Jan 2026 08:54:27 +0530 Subject: [PATCH 04/16] Fix tests/test_litellm/llms/bedrock/test_base_aws_llm.py --- tests/test_litellm/llms/bedrock/test_base_aws_llm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py index f5856cd12d6..77eda432513 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py @@ -582,7 +582,8 @@ def test_eks_irsa_ambient_credentials_used(): ) # Should create STS client without explicit credentials (using ambient credentials) - mock_boto3_client.assert_called_once_with("sts") + # Note: verify parameter is passed for SSL verification + mock_boto3_client.assert_called_once_with("sts", verify=True) # Should call assume_role mock_sts_client.assume_role.assert_called_once_with( @@ -637,11 +638,13 @@ def test_explicit_credentials_used_when_provided(): ) # Should create STS client with explicit credentials + # Note: verify parameter is passed for SSL verification mock_boto3_client.assert_called_once_with( "sts", aws_access_key_id="explicit-access-key", aws_secret_access_key="explicit-secret-key", aws_session_token="assumed-session-token", + verify=True, ) # Should call assume_role @@ -701,6 +704,7 @@ def test_partial_credentials_still_use_ambient(): aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key=None, aws_session_token=None, + verify=True, ) # Should still call assume_role @@ -748,7 +752,7 @@ def test_cross_account_role_assumption(): ) # Should use ambient credentials - mock_boto3_client.assert_called_once_with("sts") + mock_boto3_client.assert_called_once_with("sts", verify=True) # Should call assume_role with cross-account role mock_sts_client.assume_role.assert_called_once_with( From dabb459d2bc3fdea8441704f483f329cebde71a2 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 12 Jan 2026 10:57:17 +0530 Subject: [PATCH 05/16] Fix : model id encoding for bedrock passthrough --- .../bedrock/passthrough/transformation.py | 39 ++++- ...test_bedrock_passthrough_transformation.py | 139 +++++++++++++++++- 2 files changed, 169 insertions(+), 9 deletions(-) diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index 568fe941716..71c418757bf 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -24,6 +24,37 @@ class BedrockPassthroughConfig( def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: return "stream" in endpoint + def _encode_model_id_for_endpoint(self, model_id: str) -> str: + """ + Encode model_id (especially ARNs) for use in Bedrock endpoints. + + ARNs contain special characters like colons and slashes that need to be + properly URL-encoded when used in HTTP request paths. For example: + arn:aws:bedrock:us-east-1:123:application-inference-profile/abc123 + becomes: + arn:aws:bedrock:us-east-1:123:application-inference-profile%2Fabc123 + + Args: + model_id: The model ID or ARN to encode + + Returns: + The encoded model_id suitable for use in endpoint URLs + """ + from litellm.passthrough.utils import CommonUtils + import re + + # Create a temporary endpoint with the model_id to check if encoding is needed + temp_endpoint = f"/model/{model_id}/converse" + encoded_temp_endpoint = CommonUtils.encode_bedrock_runtime_modelid_arn(temp_endpoint) + + # Extract the encoded model_id from the temporary endpoint + encoded_model_id_match = re.search(r'/model/([^/]+)/', encoded_temp_endpoint) + if encoded_model_id_match: + return encoded_model_id_match.group(1) + else: + # Fallback to original model_id if extraction fails + return model_id + def get_complete_url( self, api_base: Optional[str], @@ -53,9 +84,13 @@ class BedrockPassthroughConfig( # If model_id is provided (e.g., Application Inference Profile ARN), use it in the endpoint # instead of the translated model name if model_id is not None: - # Replace the model name in the endpoint with the model_id import re - endpoint = re.sub(r'model/[^/]+/', f'model/{model_id}/', endpoint) + + # Encode the model_id if it's an ARN to properly handle special characters + encoded_model_id = self._encode_model_id_for_endpoint(model_id) + + # Replace the model name in the endpoint with the encoded model_id + endpoint = re.sub(r'model/[^/]+/', f'model/{encoded_model_id}/', endpoint) return self.format_url(endpoint, endpoint_url, request_query_params or {}), endpoint_url def sign_request( diff --git a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index 07253a8e09e..76fe0d7568a 100644 --- a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -181,7 +181,7 @@ def test_bedrock_passthrough_with_application_inference_profile(): This test verifies the fix for GitHub issue #18761 where Bedrock passthrough was not working with Application Inference Profiles. The model_id (ARN) should - replace the translated model name in the endpoint URL. + replace the translated model name in the endpoint URL and be properly encoded. """ config = BedrockPassthroughConfig() @@ -204,19 +204,21 @@ def test_bedrock_passthrough_with_application_inference_profile(): litellm_params={"model_id": model_id, "aws_region_name": "eu-west-1"} ) - # Verify that the URL contains the model_id (ARN) instead of the model name + # Verify that the URL contains the encoded model_id (ARN) instead of the model name url_str = str(url) - assert model_id in url_str, f"Expected model_id ARN in URL, but got: {url_str}" + # The ARN slash should be encoded as %2F + assert "application-inference-profile%2F" in url_str, f"Expected encoded ARN in URL, but got: {url_str}" assert model not in url_str, f"Model name should be replaced by model_id, but got: {url_str}" assert "/invoke" in url_str, "Expected /invoke action in URL" - # Verify the complete URL structure - expected_url = f"https://bedrock-runtime.eu-west-1.amazonaws.com/model/{model_id}/invoke" + # Verify the complete URL structure with encoded ARN + encoded_model_id = "arn:aws:bedrock:eu-west-1:123456789:application-inference-profile%2Fabcdefgh1234" + expected_url = f"https://bedrock-runtime.eu-west-1.amazonaws.com/model/{encoded_model_id}/invoke" assert url_str == expected_url, f"Expected {expected_url}, but got: {url_str}" def test_bedrock_passthrough_with_inference_profile_converse_endpoint(): - """Test Application Inference Profile with converse endpoint""" + """Test Application Inference Profile with converse endpoint and proper ARN encoding""" config = BedrockPassthroughConfig() model = "anthropic.claude-sonnet-4-20250514-v1:0" @@ -239,7 +241,8 @@ def test_bedrock_passthrough_with_inference_profile_converse_endpoint(): ) url_str = str(url) - assert model_id in url_str + # The ARN should be encoded with %2F + assert "application-inference-profile%2F" in url_str assert "/converse" in url_str assert model not in url_str @@ -304,3 +307,125 @@ def test_bedrock_passthrough_region_extraction_from_inference_profile_arn(): # Verify that the region from ARN is used in the base URL assert "us-west-2" in api_base, f"Expected region 'us-west-2' from ARN in base URL, but got: {api_base}" + +def test_bedrock_passthrough_model_id_arn_encoding(): + """ + Test that model_id ARNs are properly URL-encoded when used in endpoints. + + This is the critical fix for the issue where ARNs with slashes need to be encoded + so they're treated as a single path component rather than multiple path segments. + + For example: + arn:aws:bedrock:us-east-1:590183661440:application-inference-profile/b943q2qbl3m7 + should become: + arn:aws:bedrock:us-east-1:590183661440:application-inference-profile%2Fb943q2qbl3m7 + """ + config = BedrockPassthroughConfig() + + model = "bedrock-claude-4-5-sonnet" + # ARN with a slash that needs encoding + model_id = "arn:aws:bedrock:us-east-1:590183661440:application-inference-profile/b943q2qbl3m7" + endpoint = f"/model/{model}/converse" + + with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ + patch.object(config, 'get_runtime_endpoint', return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com" + )): + + url, api_base = config.get_complete_url( + api_base=None, + api_key=None, + model=model, + endpoint=endpoint, + request_query_params=None, + litellm_params={"model_id": model_id} + ) + + url_str = str(url) + + # The slash in the ARN after application-inference-profile should be encoded as %2F + assert "application-inference-profile%2F" in url_str, \ + f"Expected encoded ARN with %2F in URL, but got: {url_str}" + + # The unencoded version should NOT be in the URL + assert "application-inference-profile/" not in url_str, \ + f"ARN slash should be encoded, but found unencoded version in: {url_str}" + + # Verify the complete expected URL structure + expected_encoded_model_id = "arn:aws:bedrock:us-east-1:590183661440:application-inference-profile%2Fb943q2qbl3m7" + expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{expected_encoded_model_id}/converse" + assert url_str == expected_url, f"Expected {expected_url}, but got: {url_str}" + + +def test_bedrock_passthrough_model_id_arn_encoding_invoke_endpoint(): + """ + Test ARN encoding with /invoke endpoint (not just /converse). + """ + config = BedrockPassthroughConfig() + + model = "anthropic.claude-sonnet-4-5-20250929-v1:0" + model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz789" + endpoint = f"/model/{model}/invoke" + + with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ + patch.object(config, 'get_runtime_endpoint', return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com" + )): + + url, api_base = config.get_complete_url( + api_base=None, + api_key=None, + model=model, + endpoint=endpoint, + request_query_params=None, + litellm_params={"model_id": model_id} + ) + + url_str = str(url) + + # Verify encoding + assert "application-inference-profile%2F" in url_str + assert "/invoke" in url_str + + expected_encoded_model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile%2Fxyz789" + expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{expected_encoded_model_id}/invoke" + assert url_str == expected_url + + +def test_bedrock_passthrough_model_id_without_arn(): + """ + Test that non-ARN model_ids (regular model IDs) are not affected by encoding logic. + """ + config = BedrockPassthroughConfig() + + model = "my-model" + # Regular model ID (not an ARN) + model_id = "us.anthropic.claude-3-5-sonnet-20240620-v1:0" + endpoint = f"/model/{model}/converse" + + with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ + patch.object(config, 'get_runtime_endpoint', return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com" + )): + + url, api_base = config.get_complete_url( + api_base=None, + api_key=None, + model=model, + endpoint=endpoint, + request_query_params=None, + litellm_params={"model_id": model_id} + ) + + url_str = str(url) + + # Regular model ID should be used as-is (no encoding needed) + assert model_id in url_str + assert "%2F" not in url_str, "Non-ARN model IDs should not be encoded" + + expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse" + assert url_str == expected_url + From 0e960df8f47304d6122093ae528c04eba3c66aaa Mon Sep 17 00:00:00 2001 From: yogeshwaran10 Date: Mon, 12 Jan 2026 11:20:13 +0530 Subject: [PATCH 06/16] refactor(tests): move gemini token usage tests to test_vertex_and_google_ai_studio_gemini.py --- .../gemini/test_gemini_token_usage.py | 70 ------------------- ...test_vertex_and_google_ai_studio_gemini.py | 69 ++++++++++++++++++ 2 files changed, 69 insertions(+), 70 deletions(-) delete mode 100644 tests/test_litellm/llms/vertex_ai/gemini/test_gemini_token_usage.py diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_gemini_token_usage.py b/tests/test_litellm/llms/vertex_ai/gemini/test_gemini_token_usage.py deleted file mode 100644 index c8894ca7d5c..00000000000 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_gemini_token_usage.py +++ /dev/null @@ -1,70 +0,0 @@ - -import sys, os -import pytest -sys.path.insert(0, os.path.abspath('../../../../../')) - -from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig -from litellm.types.llms.vertex_ai import UsageMetadata - -def test_gemini_3_flash_preview_token_usage_fallback(): - """Test fallback logic when candidatesTokensDetails is missing (e.g. Gemini 3 Flash Preview).""" - v = VertexGeminiConfig() - - usage_metadata_dict = { - "promptTokenCount": 2145, - "candidatesTokenCount": 509, - "totalTokenCount": 2654, - # candidatesTokensDetails intentionally omitted - } - - completion_response = {"usageMetadata": usage_metadata_dict} - result = v._calculate_usage(completion_response=completion_response) - - assert result.completion_tokens == 509 - assert result.prompt_tokens == 2145 - assert result.total_tokens == 2654 - - # Text tokens should be derived from candidatesTokenCount - assert result.completion_tokens_details is not None - assert result.completion_tokens_details.text_tokens == 509 - assert result.completion_tokens_details.image_tokens is None - assert result.completion_tokens_details.audio_tokens is None - -def test_gemini_no_reasoning_fallback(): - """Test fallback when reasoning_effort is absent and details are missing.""" - v = VertexGeminiConfig() - - usage_metadata_dict = { - "promptTokenCount": 100, - "candidatesTokenCount": 264, - "totalTokenCount": 364, - } - - completion_response = {"usageMetadata": usage_metadata_dict} - result = v._calculate_usage(completion_response=completion_response) - - assert result.completion_tokens == 264 - assert result.completion_tokens_details is not None - assert result.completion_tokens_details.text_tokens == 264 - assert result.completion_tokens_details.reasoning_tokens is None or result.completion_tokens_details.reasoning_tokens == 0 - -def test_gemini_token_usage_standard_response(): - """Verify that standard responses with details are computed correctly and not overwritten.""" - v = VertexGeminiConfig() - - usage_metadata_dict = { - "promptTokenCount": 100, - "candidatesTokenCount": 50, - "totalTokenCount": 150, - "candidatesTokensDetails": [ - {"modality": "TEXT", "tokenCount": 40}, - {"modality": "IMAGE", "tokenCount": 10} - ] - } - - completion_response = {"usageMetadata": usage_metadata_dict} - result = v._calculate_usage(completion_response=completion_response) - - assert result.completion_tokens == 50 - assert result.completion_tokens_details.text_tokens == 40 - assert result.completion_tokens_details.image_tokens == 10 diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index d09de3a0f26..5d91275fa8d 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -2505,3 +2505,72 @@ def test_vertex_ai_multiple_function_declarations_grouped(): func_names = [f["name"] for f in tools[0]["function_declarations"]] assert "func1" in func_names assert "func2" in func_names + + +def test_gemini_3_flash_preview_token_usage_fallback(): + """Test fallback logic when candidatesTokensDetails is missing (e.g. Gemini 3 Flash Preview).""" + v = VertexGeminiConfig() + + usage_metadata_dict = { + "promptTokenCount": 2145, + "candidatesTokenCount": 509, + "totalTokenCount": 2654, + # candidatesTokensDetails intentionally omitted + } + + completion_response = {"usageMetadata": usage_metadata_dict} + result = v._calculate_usage(completion_response=completion_response) + + assert result.completion_tokens == 509 + assert result.prompt_tokens == 2145 + assert result.total_tokens == 2654 + + # Text tokens should be derived from candidatesTokenCount + assert result.completion_tokens_details is not None + assert result.completion_tokens_details.text_tokens == 509 + assert result.completion_tokens_details.image_tokens is None + assert result.completion_tokens_details.audio_tokens is None + + +def test_gemini_no_reasoning_fallback(): + """Test fallback when reasoning_effort is absent and details are missing.""" + v = VertexGeminiConfig() + + usage_metadata_dict = { + "promptTokenCount": 100, + "candidatesTokenCount": 264, + "totalTokenCount": 364, + } + + completion_response = {"usageMetadata": usage_metadata_dict} + result = v._calculate_usage(completion_response=completion_response) + + assert result.completion_tokens == 264 + assert result.completion_tokens_details is not None + assert result.completion_tokens_details.text_tokens == 264 + assert ( + result.completion_tokens_details.reasoning_tokens is None + or result.completion_tokens_details.reasoning_tokens == 0 + ) + + +def test_gemini_token_usage_standard_response(): + """Verify that standard responses with details are computed correctly and not overwritten.""" + v = VertexGeminiConfig() + + usage_metadata_dict = { + "promptTokenCount": 100, + "candidatesTokenCount": 50, + "totalTokenCount": 150, + "candidatesTokensDetails": [ + {"modality": "TEXT", "tokenCount": 40}, + {"modality": "IMAGE", "tokenCount": 10}, + ], + } + + completion_response = {"usageMetadata": usage_metadata_dict} + result = v._calculate_usage(completion_response=completion_response) + + assert result.completion_tokens == 50 + assert result.completion_tokens_details.text_tokens == 40 + assert result.completion_tokens_details.image_tokens == 10 From 1169be44b5d7e3ed8c4a8a160ab31cfa97271488 Mon Sep 17 00:00:00 2001 From: Jonathan Hoyt Date: Mon, 12 Jan 2026 02:53:58 -0800 Subject: [PATCH 07/16] fix(google_genai): forward extra_headers in generateContent adapter (#18935) When using the generateContent endpoint with non-Google providers like github_copilot, the extra_headers from model config were not being forwarded to the underlying litellm.completion/acompletion calls. This caused providers requiring custom headers (e.g., Editor-Version for GitHub Copilot authentication) to reject requests with errors like "missing Editor-Version header for IDE auth". Changes: - Forward extra_headers in _prepare_completion_kwargs() handler - Pass extra_headers explicitly to adapter in generate_content() - Pass extra_headers explicitly to adapter in agenerate_content_stream() - Pass extra_headers explicitly to adapter in generate_content_stream() - Add tests for extra_headers forwarding behavior - Update existing test to expect extra_headers in passed fields Co-authored-by: Claude --- litellm/google_genai/adapters/handler.py | 11 ++- litellm/google_genai/main.py | 3 + .../google_genai/test_google_genai_adapter.py | 5 +- .../test_google_genai_adapter_fixes.py | 76 +++++++++++++++++-- 4 files changed, 85 insertions(+), 10 deletions(-) diff --git a/litellm/google_genai/adapters/handler.py b/litellm/google_genai/adapters/handler.py index 575c36b946a..209e03d2bda 100644 --- a/litellm/google_genai/adapters/handler.py +++ b/litellm/google_genai/adapters/handler.py @@ -37,9 +37,14 @@ class GenerateContentToCompletionHandler: completion_kwargs: Dict[str, Any] = dict(completion_request) - # feed metadata for custom callback - if extra_kwargs is not None and "metadata" in extra_kwargs: - completion_kwargs["metadata"] = extra_kwargs["metadata"] + # Forward extra_kwargs that should be passed to completion call + if extra_kwargs is not None: + # Forward metadata for custom callback + if "metadata" in extra_kwargs: + completion_kwargs["metadata"] = extra_kwargs["metadata"] + # Forward extra_headers for providers that require custom headers (e.g., github_copilot) + if "extra_headers" in extra_kwargs: + completion_kwargs["extra_headers"] = extra_kwargs["extra_headers"] if stream: completion_kwargs["stream"] = stream diff --git a/litellm/google_genai/main.py b/litellm/google_genai/main.py index 1dc805a6b54..9ec56c37170 100644 --- a/litellm/google_genai/main.py +++ b/litellm/google_genai/main.py @@ -330,6 +330,7 @@ def generate_content( tools=tools, _is_async=_is_async, litellm_params=setup_result.litellm_params, + extra_headers=extra_headers, **kwargs, ) @@ -422,6 +423,7 @@ async def agenerate_content_stream( litellm_params=setup_result.litellm_params, tools=tools, stream=True, + extra_headers=extra_headers, **kwargs, ) ) @@ -507,6 +509,7 @@ def generate_content_stream( _is_async=_is_async, litellm_params=setup_result.litellm_params, stream=True, + extra_headers=extra_headers, **kwargs, ) diff --git a/tests/test_litellm/google_genai/test_google_genai_adapter.py b/tests/test_litellm/google_genai/test_google_genai_adapter.py index 135881ad209..884e06fdbc0 100644 --- a/tests/test_litellm/google_genai/test_google_genai_adapter.py +++ b/tests/test_litellm/google_genai/test_google_genai_adapter.py @@ -1120,12 +1120,13 @@ async def test_google_generate_content_with_openai(): # Print the response for verification print(f"Response: {response}") - ######################################################### + ######################################################### # validate only expected fields were sent to litellm.completion passed_fields = set(call_kwargs.keys()) # remove any GenericLiteLLMParams fields passed_fields = passed_fields - set(GenericLiteLLMParams.model_fields.keys()) - assert passed_fields == set(["model", "messages"]), f"Expected only model and messages to be passed through, got {passed_fields}" + # extra_headers is now explicitly passed through for providers that need custom headers + assert passed_fields == set(["model", "messages", "extra_headers"]), f"Expected model, messages, and extra_headers to be passed through, got {passed_fields}" @pytest.mark.asyncio async def test_agenerate_content_x_goog_api_key_header(): """ diff --git a/tests/test_litellm/google_genai/test_google_genai_adapter_fixes.py b/tests/test_litellm/google_genai/test_google_genai_adapter_fixes.py index d4d0ba9d44c..b45064003c6 100644 --- a/tests/test_litellm/google_genai/test_google_genai_adapter_fixes.py +++ b/tests/test_litellm/google_genai/test_google_genai_adapter_fixes.py @@ -252,7 +252,7 @@ def test_stream_transformation_error_handling(): def test_non_stream_response_when_stream_requested(): """Test handling of non-stream responses when streaming was requested""" from litellm.types.utils import Choices - + # Mock a non-stream response (ModelResponse with valid choices) mock_response = ModelResponse( id="test-123", @@ -270,13 +270,13 @@ def test_non_stream_response_when_stream_requested(): model="gpt-3.5-turbo", object="chat.completion" ) - + # Create an instance of the adapter adapter = GoogleGenAIAdapter() - + # Test the adapter's translate_completion_to_generate_content method directly result = adapter.translate_completion_to_generate_content(mock_response) - + # Verify the result is a valid Google GenAI format response assert "candidates" in result assert isinstance(result["candidates"], list) @@ -287,4 +287,70 @@ def test_non_stream_response_when_stream_requested(): assert isinstance(candidate["content"]["parts"], list) assert len(candidate["content"]["parts"]) > 0 assert "text" in candidate["content"]["parts"][0] - assert candidate["content"]["parts"][0]["text"] == "Hello, world!" \ No newline at end of file + assert candidate["content"]["parts"][0]["text"] == "Hello, world!" + + +def test_extra_headers_forwarding(): + """Test that extra_headers is correctly forwarded to completion call. + + This is important for providers like github_copilot that require custom + headers (e.g., Editor-Version) for authentication. + """ + # Test that extra_headers is included in completion kwargs + model = "gpt-3.5-turbo" + contents = {"role": "user", "parts": [{"text": "Test"}]} + config = {"temperature": 0.7} + + extra_kwargs = { + "extra_headers": { + "Editor-Version": "vscode/1.95.0", + "Editor-Plugin-Version": "copilot-chat/0.22.4", + "Custom-Header": "custom-value" + }, + "metadata": {"user_id": "test-user"} + } + + completion_kwargs = GenerateContentToCompletionHandler._prepare_completion_kwargs( + model=model, + contents=contents, + config=config, + stream=False, + extra_kwargs=extra_kwargs + ) + + # Verify extra_headers is forwarded + assert "extra_headers" in completion_kwargs, "extra_headers should be forwarded to completion call" + assert completion_kwargs["extra_headers"]["Editor-Version"] == "vscode/1.95.0" + assert completion_kwargs["extra_headers"]["Editor-Plugin-Version"] == "copilot-chat/0.22.4" + assert completion_kwargs["extra_headers"]["Custom-Header"] == "custom-value" + + # Verify metadata is also forwarded (existing behavior) + assert "metadata" in completion_kwargs + assert completion_kwargs["metadata"]["user_id"] == "test-user" + + +def test_extra_headers_not_present(): + """Test that missing extra_headers doesn't cause issues.""" + model = "gpt-3.5-turbo" + contents = {"role": "user", "parts": [{"text": "Test"}]} + config = {"temperature": 0.7} + + # extra_kwargs without extra_headers + extra_kwargs = { + "metadata": {"user_id": "test-user"} + } + + completion_kwargs = GenerateContentToCompletionHandler._prepare_completion_kwargs( + model=model, + contents=contents, + config=config, + stream=False, + extra_kwargs=extra_kwargs + ) + + # Verify extra_headers is not present (no error) + assert "extra_headers" not in completion_kwargs + + # Verify metadata is still forwarded + assert "metadata" in completion_kwargs + assert completion_kwargs["metadata"]["user_id"] == "test-user" \ No newline at end of file From f9430d1cabe9ad4fdfb1c19d346f1e4304487822 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Mon, 12 Jan 2026 16:46:52 +0530 Subject: [PATCH 08/16] docs: add Redis requirement warning for high-traffic deployments (#18892) --- docs/my-website/docs/proxy/db_deadlocks.md | 6 ++++++ docs/my-website/docs/proxy/deploy.md | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/docs/my-website/docs/proxy/db_deadlocks.md b/docs/my-website/docs/proxy/db_deadlocks.md index ef9d31d6232..fd02ce50e83 100644 --- a/docs/my-website/docs/proxy/db_deadlocks.md +++ b/docs/my-website/docs/proxy/db_deadlocks.md @@ -4,6 +4,12 @@ import TabItem from '@theme/TabItem'; # High Availability Setup (Resolve DB Deadlocks) +:::tip Essential for Production + +This configuration is **required** for production deployments handling 1000+ requests per second. Without Redis configured, you may experience PostgreSQL connection exhaustion (`FATAL: sorry, too many clients already`). + +::: + Resolve any Database Deadlocks you see in high traffic by using this setup ## What causes the problem? diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 9b4bc6822c1..5686e9fd835 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -359,6 +359,26 @@ LiteLLM is compatible with several SDKs - including OpenAI SDK, Anthropic SDK, M ### Deploy with Database ##### Docker, Kubernetes, Helm Chart +:::warning High Traffic Deployments (1000+ RPS) + +If you expect high traffic (1000+ requests per second), **Redis is required** to prevent database connection exhaustion and deadlocks. + +Add this to your config: +```yaml +general_settings: + use_redis_transaction_buffer: true + +litellm_settings: + cache: true + cache_params: + type: redis + host: your-redis-host +``` + +See [Resolve DB Deadlocks](/docs/proxy/db_deadlocks) for details. + +::: + Requirements: - Need a postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), etc) Set `DATABASE_URL=postgresql://:@:/` in your env - Set a `LITELLM_MASTER_KEY`, this is your Proxy Admin key - you can use this to create other keys (🚨 must start with `sk-`) From 45ac107beeb863967d28f1dba2476c0962a7f976 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Mon, 12 Jan 2026 16:47:25 +0530 Subject: [PATCH 09/16] doc: update load balancing and routing with enable_pre_call_checks (#18888) --- docs/my-website/docs/proxy/load_balancing.md | 7 +++++++ docs/my-website/docs/routing.md | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 4cff7e5d041..42f6ef1aa51 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -264,8 +264,15 @@ model_list: model: azure/gpt-4-fallback api_key: os.environ/AZURE_API_KEY_2 order: 2 # 👈 Used when order=1 is unavailable + +router_settings: + enable_pre_call_checks: true # 👈 Required for 'order' to work ``` +:::important +The `order` parameter requires `enable_pre_call_checks: true` in `router_settings`. +::: + If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments. ### When You'll See Load Balancing in Action diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 2539f70d5bc..8ac56463d5e 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -861,9 +861,13 @@ model_list = [ }, ] -router = Router(model_list=model_list) +router = Router(model_list=model_list, enable_pre_call_checks=True) # 👈 Required for 'order' to work ``` +:::important +The `order` parameter requires `enable_pre_call_checks=True` to be set on the Router. +::: + @@ -880,6 +884,9 @@ model_list: model: azure/gpt-4-fallback api_key: os.environ/AZURE_API_KEY_2 order: 2 # 👈 Used when order=1 is unavailable + +router_settings: + enable_pre_call_checks: true # 👈 Required for 'order' to work ``` From 3257cc7129a17b2cf711225b7b2463722a84e66f Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Mon, 12 Jan 2026 16:56:16 +0530 Subject: [PATCH 10/16] doc: updated pass_through with guided param (#18886) --- docs/my-website/docs/proxy/pass_through.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/my-website/docs/proxy/pass_through.md b/docs/my-website/docs/proxy/pass_through.md index 03454004b8c..cf8168764b8 100644 --- a/docs/my-website/docs/proxy/pass_through.md +++ b/docs/my-website/docs/proxy/pass_through.md @@ -165,6 +165,7 @@ general_settings: target: string # Target URL for forwarding auth: boolean # Enable LiteLLM authentication (Enterprise) forward_headers: boolean # Forward all incoming headers + include_subpath: boolean # If true, forwards requests to sub-paths (default: false) headers: # Custom headers to add Authorization: string # Auth header for target API content-type: string # Request content type @@ -181,6 +182,23 @@ general_settings: - **LANGFUSE_PUBLIC_KEY/SECRET_KEY**: For Langfuse integration - **Custom headers**: Any additional key-value pairs +### Sub-path Routing + +By default, pass-through endpoints only match the **exact path** specified. To forward requests to sub-paths, set `include_subpath: true`: + +```yaml +general_settings: + pass_through_endpoints: + - path: "/custom-api" # Any path prefix you choose + target: "https://api.example.com" + include_subpath: true # Forward /custom-api/*, not just /custom-api +``` + +| Setting | Behavior | +|---------|----------| +| `include_subpath: false` (default) | Only `/custom-api` is forwarded | +| `include_subpath: true` | `/custom-api`, `/custom-api/v1/chat`, `/custom-api/anything` are all forwarded | + --- ## Advanced: Custom Adapters From 5ce3a56ac331833bc0756c1a1cadbdc0ba1609c4 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Mon, 12 Jan 2026 17:03:03 +0530 Subject: [PATCH 11/16] add better err handling for antropic (#18955) --- .../prompt_templates/common_utils.py | 102 +++++++--- .../prompt_templates/factory.py | 166 +++++++++++----- .../adapters/transformation.py | 20 +- tests/llm_translation/test_prompt_factory.py | 182 ++++++++++++------ 4 files changed, 330 insertions(+), 140 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 2f8568db704..a8b8b207de4 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -95,7 +95,9 @@ def handle_messages_with_content_list_to_str_conversion( return messages -def strip_name_from_message(message: AllMessageValues, allowed_name_roles: List[str] = ["user"]) -> AllMessageValues: +def strip_name_from_message( + message: AllMessageValues, allowed_name_roles: List[str] = ["user"] +) -> AllMessageValues: """ Removes 'name' from message """ @@ -104,6 +106,7 @@ def strip_name_from_message(message: AllMessageValues, allowed_name_roles: List[ msg_copy.pop("name", None) # type: ignore return msg_copy + def strip_name_from_messages( messages: List[AllMessageValues], allowed_name_roles: List[str] = ["user"] ) -> List[AllMessageValues]: @@ -444,7 +447,7 @@ def update_responses_input_with_model_file_ids( """ Updates responses API input with provider-specific file IDs. File IDs are always inside the content array, not as direct input_file items. - + For managed files (unified file IDs), decodes the base64-encoded unified file ID and extracts the llm_output_file_id directly. """ @@ -452,25 +455,28 @@ def update_responses_input_with_model_file_ids( _is_base64_encoded_unified_file_id, convert_b64_uid_to_unified_uid, ) - + if isinstance(input, str): return input - + if not isinstance(input, list): return input - + updated_input = [] for item in input: if not isinstance(item, dict): updated_input.append(item) continue - + updated_item = item.copy() content = item.get("content") if isinstance(content, list): updated_content = [] for content_item in content: - if isinstance(content_item, dict) and content_item.get("type") == "input_file": + if ( + isinstance(content_item, dict) + and content_item.get("type") == "input_file" + ): file_id = content_item.get("file_id") if file_id: # Check if this is a managed file ID (base64-encoded unified file ID) @@ -478,7 +484,9 @@ def update_responses_input_with_model_file_ids( if is_unified_file_id: unified_file_id = convert_b64_uid_to_unified_uid(file_id) if "llm_output_file_id," in unified_file_id: - provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0] + provider_file_id = unified_file_id.split( + "llm_output_file_id," + )[1].split(";")[0] else: # Fallback: keep original if we can't extract provider_file_id = file_id @@ -492,9 +500,9 @@ def update_responses_input_with_model_file_ids( else: updated_content.append(content_item) updated_item["content"] = updated_content - + updated_input.append(updated_item) - + return updated_input @@ -697,9 +705,9 @@ def _get_image_mime_type_from_url(url: str) -> Optional[str]: video/flv """ from urllib.parse import urlparse - + url = url.lower() - + # Parse URL to extract path without query parameters # This handles URLs like: https://example.com/image.jpg?signature=... parsed = urlparse(url) @@ -744,28 +752,28 @@ def infer_content_type_from_url_and_content( ) -> str: """ Infer content type from URL extension and binary content when content-type header is missing or generic. - + This helper implements a fallback strategy for determining MIME types when HTTP headers are missing or provide generic values (like binary/octet-stream). It's commonly used when processing images and documents from various sources (S3, URLs, etc.). - + Fallback Strategy: 1. If current_content_type is valid (not None and not generic octet-stream), return it 2. Try to infer from URL extension (handles query parameters) 3. Try to detect from binary content signature (magic bytes) 4. Raise ValueError if all methods fail - + Args: url: The URL of the content (used to extract file extension) content: The binary content (first ~100 bytes are sufficient for detection) current_content_type: The current content-type from headers (may be None or generic) - + Returns: str: The inferred MIME type (e.g., "image/png", "application/pdf") - + Raises: ValueError: If content type cannot be determined by any method - + Example: >>> content_type = infer_content_type_from_url_and_content( ... url="https://s3.amazonaws.com/bucket/image.png?AWSAccessKeyId=123", @@ -776,14 +784,14 @@ def infer_content_type_from_url_and_content( "image/png" """ from litellm.litellm_core_utils.token_counter import get_image_type - + # If we have a valid content type that's not generic, use it if current_content_type and current_content_type not in [ "binary/octet-stream", "application/octet-stream", ]: return current_content_type - + # Extension to MIME type mapping # Supports images, documents, and other common file types extension_to_mime = { @@ -804,14 +812,14 @@ def infer_content_type_from_url_and_content( "txt": "text/plain", "md": "text/markdown", } - + # Try to infer from URL extension if url: extension = url.split(".")[-1].lower().split("?")[0] # Remove query params inferred_type = extension_to_mime.get(extension) if inferred_type: return inferred_type - + # Try to detect from binary content signature (magic bytes) if content: detected_type = get_image_type(content[:100]) @@ -825,7 +833,7 @@ def infer_content_type_from_url_and_content( } if detected_type in type_to_mime: return type_to_mime[detected_type] - + # If all fallbacks failed, raise error raise ValueError( f"Unable to determine content type from URL: {url}. " @@ -1085,7 +1093,9 @@ def _parse_content_for_reasoning( return None, message_text reasoning_match = re.match( - r"<(?:think|thinking|budget:thinking)>(.*?)(.*)", message_text, re.DOTALL + r"<(?:think|thinking|budget:thinking)>(.*?)(.*)", + message_text, + re.DOTALL, ) if reasoning_match: @@ -1135,3 +1145,47 @@ def extract_images_from_message(message: AllMessageValues) -> List[str]: elif isinstance(image_url, dict) and "url" in image_url: images.append(_extract_base64_data(image_url["url"])) return images + + +def parse_tool_call_arguments( + arguments: Optional[str], + tool_name: Optional[str] = None, + context: Optional[str] = None, +) -> Dict[str, Any]: + """ + Parse tool call arguments from a JSON string. + + This function handles malformed JSON gracefully by raising a ValueError + with context about what failed and what the problematic input was. + + Args: + arguments: The JSON string containing tool arguments, or None. + tool_name: Optional name of the tool (for error messages). + context: Optional context string (e.g., "Anthropic Messages API"). + + Returns: + Parsed arguments as a dictionary. Returns empty dict if arguments is None or empty. + + Raises: + ValueError: If the arguments string is not valid JSON. + """ + import json + + if not arguments: + return {} + + try: + return json.loads(arguments) + except json.JSONDecodeError as e: + error_parts = ["Failed to parse tool call arguments"] + + if tool_name: + error_parts.append(f"for tool '{tool_name}'") + if context: + error_parts.append(f"({context})") + + error_message = ( + " ".join(error_parts) + f". Error: {str(e)}. Arguments: {arguments}" + ) + + raise ValueError(error_message) from e diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d8e82199272..4320f756454 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -44,6 +44,7 @@ from .common_utils import ( convert_content_list_to_str, infer_content_type_from_url_and_content, is_non_content_values_set, + parse_tool_call_arguments, ) from .image_handling import convert_url_to_base64 @@ -911,13 +912,13 @@ def convert_to_anthropic_image_obj( def create_anthropic_image_param( - image_url_input: Union[str, dict], + image_url_input: Union[str, dict], format: Optional[str] = None, - is_bedrock_invoke: bool = False + is_bedrock_invoke: bool = False, ) -> AnthropicMessagesImageParam: """ Create an AnthropicMessagesImageParam from an image URL input. - + Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding. """ # Extract URL and format from input @@ -927,7 +928,7 @@ def create_anthropic_image_param( image_url = image_url_input.get("url", "") if format is None: format = image_url_input.get("format") - + # Check if the image URL is an HTTP/HTTPS URL if image_url.startswith("http://") or image_url.startswith("https://"): # For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64 @@ -1031,9 +1032,11 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str: tool_function = get_attribute_or_key(tool, "function") tool_name = get_attribute_or_key(tool_function, "name") tool_arguments = get_attribute_or_key(tool_function, "arguments") + parsed_args = parse_tool_call_arguments( + tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke" + ) parameters = "".join( - f"<{param}>{val}\n" - for param, val in json.loads(tool_arguments).items() + f"<{param}>{val}\n" for param, val in parsed_args.items() ) invokes += ( "\n" @@ -1071,8 +1074,14 @@ def anthropic_messages_pt_xml(messages: list): if isinstance(messages[msg_i]["content"], list): for m in messages[msg_i]["content"]: if m.get("type", "") == "image_url": - format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None - image_param = create_anthropic_image_param(m["image_url"], format=format) + format = ( + m["image_url"].get("format") + if isinstance(m["image_url"], dict) + else None + ) + image_param = create_anthropic_image_param( + m["image_url"], format=format + ) # Convert to dict format for XML version source = image_param["source"] if isinstance(source, dict) and source.get("type") == "url": @@ -1381,10 +1390,10 @@ def convert_to_gemini_tool_call_invoke( if tool_calls is not None: for idx, tool in enumerate(tool_calls): if "function" in tool: - gemini_function_call: Optional[ - VertexFunctionCall - ] = _gemini_tool_call_invoke_helper( - function_call_params=tool["function"] + gemini_function_call: Optional[VertexFunctionCall] = ( + _gemini_tool_call_invoke_helper( + function_call_params=tool["function"] + ) ) if gemini_function_call is not None: part_dict: VertexPartType = { @@ -1484,10 +1493,10 @@ def convert_to_gemini_tool_call_result( } """ from litellm.types.llms.vertex_ai import BlobType - + content_str: str = "" inline_data: Optional[BlobType] = None - + if "content" in message: if isinstance(message["content"], str): content_str = message["content"] @@ -1500,15 +1509,21 @@ def convert_to_gemini_tool_call_result( elif content_type in ("input_image", "image_url"): # Extract image for inline_data (for Computer Use screenshots and tool results) image_url_data = content.get("image_url", "") - image_url = image_url_data.get("url", "") if isinstance(image_url_data, dict) else image_url_data - + image_url = ( + image_url_data.get("url", "") + if isinstance(image_url_data, dict) + else image_url_data + ) + if image_url: # Convert image to base64 blob format for Gemini try: - image_obj = convert_to_anthropic_image_obj(image_url, format=None) + image_obj = convert_to_anthropic_image_obj( + image_url, format=None + ) inline_data = BlobType( data=image_obj["data"], - mime_type=image_obj["media_type"] + mime_type=image_obj["media_type"], ) except Exception as e: verbose_logger.warning( @@ -1541,6 +1556,7 @@ def convert_to_gemini_tool_call_result( response_data: dict try: import json + if content_str.strip().startswith("{") or content_str.strip().startswith("["): # Try to parse as JSON (for Computer Use structured responses) parsed = json.loads(content_str) @@ -1553,7 +1569,7 @@ def convert_to_gemini_tool_call_result( except (json.JSONDecodeError, ValueError): # Not valid JSON, wrap in content field response_data = {"content": content_str} - + # We can't determine from openai message format whether it's a successful or # error call result so default to the successful result template _function_response = VertexFunctionResponse( @@ -1562,7 +1578,7 @@ def convert_to_gemini_tool_call_result( # Create part with function_response, and optionally inline_data for images (Computer Use) _part: VertexPartType = {"function_response": _function_response} - + # For Computer Use, if we have an image, we need separate parts: # - One part with function_response # - One part with inline_data @@ -1570,19 +1586,19 @@ def convert_to_gemini_tool_call_result( if inline_data: image_part: VertexPartType = {"inline_data": inline_data} return [_part, image_part] - + return _part def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: """ Sanitize tool_use_id to match Anthropic's required pattern: ^[a-zA-Z0-9_-]+$ - + Anthropic requires tool_use_id to only contain alphanumeric characters, underscores, and hyphens. This function replaces any invalid characters with underscores. """ # Replace any character that's not alphanumeric, underscore, or hyphen with underscore - sanitized = re.sub(r'[^a-zA-Z0-9_-]', '_', tool_use_id) + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", tool_use_id) # Ensure it's not empty (fallback to a default if needed) if not sanitized: sanitized = "tool_use_id" @@ -1644,8 +1660,14 @@ def convert_to_anthropic_tool_result( ) ) elif content["type"] == "image_url": - format = content["image_url"].get("format") if isinstance(content["image_url"], dict) else None - _anthropic_image_param = create_anthropic_image_param(content["image_url"], format=format) + format = ( + content["image_url"].get("format") + if isinstance(content["image_url"], dict) + else None + ) + _anthropic_image_param = create_anthropic_image_param( + content["image_url"], format=format + ) _anthropic_image_param = add_cache_control_to_content( anthropic_content_element=_anthropic_image_param, original_content_element=content, @@ -1665,7 +1687,9 @@ def convert_to_anthropic_tool_result( # We can't determine from openai message format whether it's a successful or # error call result so default to the successful result template anthropic_tool_result = AnthropicMessagesToolResultParam( - type="tool_result", tool_use_id=sanitized_tool_use_id, content=anthropic_content + type="tool_result", + tool_use_id=sanitized_tool_use_id, + content=anthropic_content, ) if message["role"] == "function": @@ -1674,7 +1698,9 @@ def convert_to_anthropic_tool_result( # Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$ sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id) anthropic_tool_result = AnthropicMessagesToolResultParam( - type="tool_result", tool_use_id=sanitized_tool_use_id, content=anthropic_content + type="tool_result", + tool_use_id=sanitized_tool_use_id, + content=anthropic_content, ) if anthropic_tool_result is None: @@ -1690,12 +1716,17 @@ def convert_function_to_anthropic_tool_invoke( try: _name = get_attribute_or_key(function_call, "name") or "" _arguments = get_attribute_or_key(function_call, "arguments") + + tool_input = parse_tool_call_arguments( + _arguments, tool_name=_name, context="Anthropic function to tool invoke" + ) + anthropic_tool_invoke = [ AnthropicMessagesToolUseParam( type="tool_use", id=str(uuid.uuid4()), name=_name, - input=json.loads(_arguments) if _arguments else {}, + input=tool_input, ) ] return anthropic_tool_invoke @@ -1749,7 +1780,9 @@ def convert_to_anthropic_tool_invoke( Fixes: https://github.com/BerriAI/litellm/issues/17737 """ - anthropic_tool_invoke: List[Union[AnthropicMessagesToolUseParam, Dict[str, Any]]] = [] + anthropic_tool_invoke: List[ + Union[AnthropicMessagesToolUseParam, Dict[str, Any]] + ] = [] for tool in tool_calls: if not get_attribute_or_key(tool, "type") == "function": @@ -1760,10 +1793,10 @@ def convert_to_anthropic_tool_invoke( str, get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"), ) - tool_input = json.loads( - get_attribute_or_key( - get_attribute_or_key(tool, "function"), "arguments" - ) + tool_input = parse_tool_call_arguments( + get_attribute_or_key(get_attribute_or_key(tool, "function"), "arguments"), + tool_name=tool_name, + context="Anthropic tool invoke", ) # Check if this is a server-side tool (web_search, tool_search, etc.) @@ -2015,11 +2048,17 @@ def anthropic_messages_pt( # noqa: PLR0915 for m in user_message_types_block["content"]: if m.get("type", "") == "image_url": m = cast(ChatCompletionImageObject, m) - format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None + format = ( + m["image_url"].get("format") + if isinstance(m["image_url"], dict) + else None + ) # Convert ChatCompletionImageUrlObject to dict if needed image_url_value = m["image_url"] if isinstance(image_url_value, str): - image_url_input: Union[str, dict[str, Any]] = image_url_value + image_url_input: Union[str, dict[str, Any]] = ( + image_url_value + ) else: # ChatCompletionImageUrlObject or dict case - convert to dict image_url_input = { @@ -2029,20 +2068,26 @@ def anthropic_messages_pt( # noqa: PLR0915 # Bedrock invoke models have format: invoke/... # Vertex AI Anthropic also doesn't support URL sources for images is_bedrock_invoke = model.lower().startswith("invoke/") - is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False + is_vertex_ai = ( + llm_provider.startswith("vertex_ai") + if llm_provider + else False + ) force_base64 = is_bedrock_invoke or is_vertex_ai _anthropic_content_element = create_anthropic_image_param( - image_url_input, format=format, is_bedrock_invoke=force_base64 - ) + image_url_input, + format=format, + is_bedrock_invoke=force_base64, + ) _content_element = add_cache_control_to_content( anthropic_content_element=_anthropic_content_element, original_content_element=dict(m), ) if "cache_control" in _content_element: - _anthropic_content_element[ - "cache_control" - ] = _content_element["cache_control"] + _anthropic_content_element["cache_control"] = ( + _content_element["cache_control"] + ) user_content.append(_anthropic_content_element) elif m.get("type", "") == "text": m = cast(ChatCompletionTextObject, m) @@ -2080,9 +2125,9 @@ def anthropic_messages_pt( # noqa: PLR0915 ) if "cache_control" in _content_element: - _anthropic_content_text_element[ - "cache_control" - ] = _content_element["cache_control"] + _anthropic_content_text_element["cache_control"] = ( + _content_element["cache_control"] + ) user_content.append(_anthropic_content_text_element) @@ -2178,18 +2223,27 @@ def anthropic_messages_pt( # noqa: PLR0915 ): # support assistant tool invoke conversion # Get web_search_results from provider_specific_fields for server_tool_use reconstruction # Fixes: https://github.com/BerriAI/litellm/issues/17737 - _provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields") + _provider_specific_fields_raw = assistant_content_block.get( + "provider_specific_fields" + ) _provider_specific_fields: Dict[str, Any] = {} if isinstance(_provider_specific_fields_raw, dict): - _provider_specific_fields = cast(Dict[str, Any], _provider_specific_fields_raw) - _web_search_results = _provider_specific_fields.get("web_search_results") + _provider_specific_fields = cast( + Dict[str, Any], _provider_specific_fields_raw + ) + _web_search_results = _provider_specific_fields.get( + "web_search_results" + ) tool_invoke_results = convert_to_anthropic_tool_invoke( assistant_tool_calls, web_search_results=_web_search_results, ) # AnthropicMessagesAssistantMessageValues includes AnthropicMessagesToolUseParam assistant_content.extend( - cast(List[AnthropicMessagesAssistantMessageValues], tool_invoke_results) + cast( + List[AnthropicMessagesAssistantMessageValues], + tool_invoke_results, + ) ) assistant_function_call = assistant_content_block.get("function_call") @@ -3252,14 +3306,18 @@ def _convert_to_bedrock_tool_call_result( """ - """ - tool_result_content_blocks:List[BedrockToolResultContentBlock] = [] + tool_result_content_blocks: List[BedrockToolResultContentBlock] = [] if isinstance(message["content"], str): - tool_result_content_blocks.append(BedrockToolResultContentBlock(text=message["content"])) + tool_result_content_blocks.append( + BedrockToolResultContentBlock(text=message["content"]) + ) elif isinstance(message["content"], List): content_list = message["content"] for content in content_list: if content["type"] == "text": - tool_result_content_blocks.append(BedrockToolResultContentBlock(text=content["text"])) + tool_result_content_blocks.append( + BedrockToolResultContentBlock(text=content["text"]) + ) elif content["type"] == "image_url": format: Optional[str] = None if isinstance(content["image_url"], dict): @@ -3267,12 +3325,14 @@ def _convert_to_bedrock_tool_call_result( format = content["image_url"].get("format") else: image_url = content["image_url"] - _block:BedrockContentBlock = BedrockImageProcessor.process_image_sync( + _block: BedrockContentBlock = BedrockImageProcessor.process_image_sync( image_url=image_url, format=format, ) if "image" in _block: - tool_result_content_blocks.append(BedrockToolResultContentBlock(image=_block["image"])) + tool_result_content_blocks.append( + BedrockToolResultContentBlock(image=_block["image"]) + ) message.get("name", "") id = str(message.get("tool_call_id", str(uuid.uuid4()))) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 8868fabdcef..06092755b17 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -14,6 +14,10 @@ from typing import ( from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingChoice +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + parse_tool_call_arguments, +) + from litellm.types.llms.anthropic import ( AllAnthropicToolsValues, AnthopicMessagesAssistantMessageParam, @@ -425,15 +429,15 @@ class LiteLLMAnthropicMessagesAdapter: ) -> Optional[str]: """ Translate Anthropic's thinking parameter to OpenAI's reasoning_effort. - + Anthropic thinking format: {'type': 'enabled'|'disabled', 'budget_tokens': int} OpenAI reasoning_effort: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'default' """ if not isinstance(thinking, dict): return None - + thinking_type = thinking.get("type", "disabled") - + if thinking_type == "disabled": return None elif thinking_type == "enabled": @@ -446,7 +450,7 @@ class LiteLLMAnthropicMessagesAdapter: return "low" else: return "minimal" - + return None def translate_anthropic_tool_choice_to_openai( @@ -676,10 +680,10 @@ class LiteLLMAnthropicMessagesAdapter: type="tool_use", id=tool_call.id, name=tool_call.function.name or "", - input=( - json.loads(tool_call.function.arguments) - if tool_call.function.arguments - else {} + input=parse_tool_call_arguments( + tool_call.function.arguments, + tool_name=tool_call.function.name, + context="Anthropic pass-through adapter", ), ) # Add provider_specific_fields if signature is present diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index d0c0c12ba7c..8974632631d 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -7,11 +7,10 @@ import pytest sys.path.insert(0, os.path.abspath("../..")) -from typing import Union, List +from typing import List # from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory import litellm -from litellm import completion from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_tools_pt, anthropic_messages_pt, @@ -31,7 +30,7 @@ from litellm.llms.vertex_ai.gemini.transformation import ( _gemini_convert_messages_with_history, ) from litellm.types.llms.openai import AllMessageValues -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch def test_llama_3_prompt(): @@ -129,10 +128,6 @@ def test_anthropic_pt_formatting(): def test_anthropic_messages_nested_pt(): - from litellm.types.llms.anthropic import ( - AnthopicMessagesAssistantMessageParam, - AnthropicMessagesUserMessageParam, - ) messages = [ {"content": [{"text": "here is a task", "type": "text"}], "role": "user"}, @@ -214,7 +209,7 @@ def test_create_anthropic_image_param_with_http_url(): image_param = create_anthropic_image_param( "https://example.com/image.jpg", format=None ) - + assert image_param["type"] == "image" assert image_param["source"]["type"] == "url" assert image_param["source"]["url"] == "https://example.com/image.jpg" @@ -225,7 +220,7 @@ def test_create_anthropic_image_param_with_https_url(): image_param = create_anthropic_image_param( "https://example.com/image.png", format=None ) - + assert image_param["type"] == "image" assert image_param["source"]["type"] == "url" assert image_param["source"]["url"] == "https://example.com/image.png" @@ -236,7 +231,7 @@ def test_create_anthropic_image_param_with_dict_input(): image_param = create_anthropic_image_param( {"url": "https://example.com/image.jpg", "format": "image/jpeg"}, format=None ) - + assert image_param["type"] == "image" assert image_param["source"]["type"] == "url" assert image_param["source"]["url"] == "https://example.com/image.jpg" @@ -247,7 +242,7 @@ def test_create_anthropic_image_param_with_base64_data_uri(): image_param = create_anthropic_image_param( "data:image/jpeg;base64,/9j/4AAQSkZJRg==", format=None ) - + assert image_param["type"] == "image" assert image_param["source"]["type"] == "base64" assert image_param["source"]["media_type"] == "image/jpeg" @@ -259,7 +254,7 @@ def test_create_anthropic_image_param_with_format_override(): image_param = create_anthropic_image_param( "data:image/jpeg;base64,1234", format="image/png" ) - + assert image_param["type"] == "image" assert image_param["source"]["type"] == "base64" assert image_param["source"]["media_type"] == "image/png" @@ -279,19 +274,19 @@ def test_anthropic_messages_pt_with_url_image(): ], } ] - + result = anthropic_messages_pt( messages=messages, model="claude-3-5-sonnet", llm_provider="anthropic" ) - + assert len(result) == 1 assert result[0]["role"] == "user" assert isinstance(result[0]["content"], list) assert len(result[0]["content"]) == 2 - + # Check text content assert result[0]["content"][0]["type"] == "text" - + # Check image content - should be URL reference, not base64 assert result[0]["content"][1]["type"] == "image" assert result[0]["content"][1]["source"]["type"] == "url" @@ -312,16 +307,16 @@ def test_anthropic_messages_pt_with_base64_image(): ], } ] - + result = anthropic_messages_pt( messages=messages, model="claude-3-5-sonnet", llm_provider="anthropic" ) - + assert len(result) == 1 assert result[0]["role"] == "user" assert isinstance(result[0]["content"], list) assert len(result[0]["content"]) == 2 - + # Check image content - should be base64, not URL assert result[0]["content"][1]["type"] == "image" assert result[0]["content"][1]["source"]["type"] == "base64" @@ -568,7 +563,9 @@ def test_vertex_only_image_user_message(): }, ] - response = _gemini_convert_messages_with_history(messages=messages, model="gemini-1.5-pro") + response = _gemini_convert_messages_with_history( + messages=messages, model="gemini-1.5-pro" + ) expected_response = [ { @@ -962,8 +959,8 @@ def test_convert_to_anthropic_tool_invoke_regular_tool(): "type": "function", "function": { "name": "get_weather", - "arguments": '{"location": "San Francisco"}' - } + "arguments": '{"location": "San Francisco"}', + }, } ] @@ -979,7 +976,7 @@ def test_convert_to_anthropic_tool_invoke_regular_tool(): def test_convert_to_anthropic_tool_invoke_server_tool(): """ Test that server_tool_use (srvtoolu_) is reconstructed as server_tool_use. - + Fixes: https://github.com/BerriAI/litellm/issues/17737 """ tool_calls = [ @@ -988,8 +985,8 @@ def test_convert_to_anthropic_tool_invoke_server_tool(): "type": "function", "function": { "name": "web_search", - "arguments": '{"query": "elephant weight"}' - } + "arguments": '{"query": "elephant weight"}', + }, } ] @@ -1005,7 +1002,7 @@ def test_convert_to_anthropic_tool_invoke_server_tool(): def test_convert_to_anthropic_tool_invoke_with_web_search_results(): """ Test that web_search_tool_result is included after server_tool_use. - + Fixes: https://github.com/BerriAI/litellm/issues/17737 """ tool_calls = [ @@ -1014,8 +1011,8 @@ def test_convert_to_anthropic_tool_invoke_with_web_search_results(): "type": "function", "function": { "name": "web_search", - "arguments": '{"query": "elephant weight"}' - } + "arguments": '{"query": "elephant weight"}', + }, } ] @@ -1028,13 +1025,15 @@ def test_convert_to_anthropic_tool_invoke_with_web_search_results(): "type": "web_search_result", "url": "https://example.com", "title": "Elephant Facts", - "snippet": "Elephants weigh 5000 kg" + "snippet": "Elephants weigh 5000 kg", } - ] + ], } ] - result = convert_to_anthropic_tool_invoke(tool_calls, web_search_results=web_search_results) + result = convert_to_anthropic_tool_invoke( + tool_calls, web_search_results=web_search_results + ) assert len(result) == 2 # First: server_tool_use @@ -1048,7 +1047,7 @@ def test_convert_to_anthropic_tool_invoke_with_web_search_results(): def test_convert_to_anthropic_tool_invoke_mixed_tools(): """ Test that mixed server and regular tools are reconstructed correctly. - + Fixes: https://github.com/BerriAI/litellm/issues/17737 """ tool_calls = [ @@ -1057,28 +1056,27 @@ def test_convert_to_anthropic_tool_invoke_mixed_tools(): "type": "function", "function": { "name": "web_search", - "arguments": '{"query": "elephant weight"}' - } + "arguments": '{"query": "elephant weight"}', + }, }, { "id": "toolu_01XYZ789", "type": "function", - "function": { - "name": "add_numbers", - "arguments": '{"a": 5000, "b": 100}' - } - } + "function": {"name": "add_numbers", "arguments": '{"a": 5000, "b": 100}'}, + }, ] web_search_results = [ { "type": "web_search_tool_result", "tool_use_id": "srvtoolu_01ABC123", - "content": [{"url": "https://example.com", "title": "Test"}] + "content": [{"url": "https://example.com", "title": "Test"}], } ] - result = convert_to_anthropic_tool_invoke(tool_calls, web_search_results=web_search_results) + result = convert_to_anthropic_tool_invoke( + tool_calls, web_search_results=web_search_results + ) assert len(result) == 3 # First: server_tool_use @@ -1094,7 +1092,7 @@ def test_convert_to_anthropic_tool_invoke_mixed_tools(): def test_anthropic_messages_pt_with_server_tool_use(): """ Test that anthropic_messages_pt correctly reconstructs server_tool_use from provider_specific_fields. - + Fixes: https://github.com/BerriAI/litellm/issues/17737 """ messages = [ @@ -1108,36 +1106,40 @@ def test_anthropic_messages_pt_with_server_tool_use(): "type": "function", "function": { "name": "web_search", - "arguments": '{"query": "elephant weight"}' - } + "arguments": '{"query": "elephant weight"}', + }, }, { "id": "toolu_01XYZ789", "type": "function", "function": { "name": "add_numbers", - "arguments": '{"a": 5000, "b": 100}' - } - } + "arguments": '{"a": 5000, "b": 100}', + }, + }, ], "provider_specific_fields": { "web_search_results": [ { "type": "web_search_tool_result", "tool_use_id": "srvtoolu_01ABC123", - "content": [{"url": "https://example.com", "title": "Test", "snippet": "5000 kg"}] + "content": [ + { + "url": "https://example.com", + "title": "Test", + "snippet": "5000 kg", + } + ], } ] - } + }, }, - { - "role": "tool", - "tool_call_id": "toolu_01XYZ789", - "content": "5100" - } + {"role": "tool", "tool_call_id": "toolu_01XYZ789", "content": "5100"}, ] - result = anthropic_messages_pt(messages, model="claude-sonnet-4-5", llm_provider="anthropic") + result = anthropic_messages_pt( + messages, model="claude-sonnet-4-5", llm_provider="anthropic" + ) # Find the assistant message assistant_msg = next(m for m in result if m["role"] == "assistant") @@ -1162,3 +1164,73 @@ def test_anthropic_messages_pt_with_server_tool_use(): # Verify regular tool_use tool_use = next(c for c in content if c.get("type") == "tool_use") assert tool_use["id"] == "toolu_01XYZ789" + + +# ============ parse_tool_call_arguments Tests ============ +# Tests for the shared utility that parses tool call JSON arguments + + +def test_parse_tool_call_arguments_valid_json(): + """Test that valid JSON is parsed correctly.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + parse_tool_call_arguments, + ) + + result = parse_tool_call_arguments('{"city": "Paris", "units": "celsius"}') + assert result == {"city": "Paris", "units": "celsius"} + + +def test_parse_tool_call_arguments_empty_input(): + """Test that None/empty input returns empty dict.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + parse_tool_call_arguments, + ) + + assert parse_tool_call_arguments(None) == {} + assert parse_tool_call_arguments("") == {} + + +def test_parse_tool_call_arguments_malformed_json(): + """Test that malformed JSON raises ValueError with context.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + parse_tool_call_arguments, + ) + + with pytest.raises(ValueError) as exc_info: + parse_tool_call_arguments( + '{"skill_name": "pptx', + tool_name="load_skill", + context="Anthropic tool invoke", + ) + + error_msg = str(exc_info.value) + assert "load_skill" in error_msg + assert "Anthropic tool invoke" in error_msg + assert '{"skill_name": "pptx' in error_msg + assert "Unterminated string" in error_msg + + +def test_convert_to_anthropic_tool_invoke_malformed_json(): + """ + Test that convert_to_anthropic_tool_invoke raises ValueError with context + when tool arguments contain malformed JSON. + + Fixes: https://github.com/BerriAI/litellm/issues/18920 + """ + tool_calls = [ + { + "id": "toolu_01_invalid", + "type": "function", + "function": { + "name": "bad_tool", + "arguments": '{"truncated', # Malformed JSON + }, + } + ] + + with pytest.raises(ValueError) as exc_info: + convert_to_anthropic_tool_invoke(tool_calls) + + error_msg = str(exc_info.value) + assert "bad_tool" in error_msg + assert '{"truncated' in error_msg From 46dd420833fbdea755187a894572d3d7598fac1f Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Mon, 12 Jan 2026 08:34:59 -0300 Subject: [PATCH 12/16] fix: sync Helm chart versioning with production standards and Docker versions (#18868) * fix: sync Helm chart versioning with production standards and Docker versions - Update Chart.yaml version from 0.4.10 to 1.0.0 (SemVer 0.x is for development, 1.0+ for production) - Update appVersion from v1.50.2 to v1.80.12 to match current Docker image version - Update workflow defaults from 0.1.0 to 1.0.0 for new chart version scheme - Maintain independent chart versioning per Helm best practices This ensures: - Helm chart follows SemVer production standards (1.x instead of 0.x) - appVersion stays synchronized with Docker/application version - Chart version remains independent for flexibility (can update chart without waiting for app releases) * fix: sync Helm chart appVersion with Docker image tags in release workflow Updates the GitHub workflow to ensure Helm chart appVersion matches the Docker image tags that are actually published: - For stable/rc releases: Uses the workflow input tag (e.g., v1.80.12) - For latest/dev releases: Uses the release_type to match main-{type} tags - Makes 'tag' input required to prevent accidental releases with wrong versions - Simplifies fallback logic by removing git-describe dependency This ensures the chart's appVersion correctly references Docker images that exist, preventing deployment failures from missing image tags. * Update ghcr_deploy.yml --- .github/workflows/ghcr_deploy.yml | 29 ++++++++++++++++++++------- deploy/charts/litellm-helm/Chart.yaml | 4 ++-- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml index f574ec9c202..aa032972b80 100644 --- a/.github/workflows/ghcr_deploy.yml +++ b/.github/workflows/ghcr_deploy.yml @@ -5,6 +5,7 @@ on: inputs: tag: description: "The tag version you want to build" + required: true release_type: description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'" type: string @@ -336,9 +337,9 @@ jobs: run: | CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true) if [ -z "${CHART_LIST}" ]; then - echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT + echo "current-version=1.0.0" | tee -a $GITHUB_OUTPUT else - # Extract version and strip any prerelease suffix (e.g., 0.1.827-latest -> 0.1.827) + # Extract version and strip any prerelease suffix (e.g., 1.0.5-latest -> 1.0.5) VERSION=$(printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print $2}' | tr -d " " | cut -d'-' -f1) echo "current-version=${VERSION}" | tee -a $GITHUB_OUTPUT fi @@ -350,28 +351,42 @@ jobs: id: bump_version uses: christian-draeger/increment-semantic-version@1.1.0 with: - current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }} + current-version: ${{ steps.current_version.outputs.current-version || '1.0.0' }} version-fragment: 'bug' # Add suffix for non-stable releases (semantic versioning) - - name: Calculate chart version with prerelease suffix + - name: Calculate chart and app versions id: chart_version shell: bash run: | - BASE_VERSION="${{ steps.bump_version.outputs.next-version || '0.1.0' }}" + BASE_VERSION="${{ steps.bump_version.outputs.next-version || '1.0.0' }}" RELEASE_TYPE="${{ github.event.inputs.release_type }}" + INPUT_TAG="${{ github.event.inputs.tag }}" + + # Chart version (independent Helm chart versioning with release type suffix) if [ "$RELEASE_TYPE" = "stable" ]; then echo "version=${BASE_VERSION}" | tee -a $GITHUB_OUTPUT else echo "version=${BASE_VERSION}-${RELEASE_TYPE}" | tee -a $GITHUB_OUTPUT fi + # App version (must match Docker tags) + # stable/rc releases: Docker creates main-{tag}, so use the tag + # latest/dev releases: Docker only creates main-{release_type}, so use release_type + if [ "$RELEASE_TYPE" = "stable" ] || [ "$RELEASE_TYPE" = "rc" ]; then + APP_VERSION="${INPUT_TAG}" + else + APP_VERSION="${RELEASE_TYPE}" + fi + + echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT + - uses: ./.github/actions/helm-oci-chart-releaser with: name: ${{ env.CHART_NAME }} repository: ${{ env.REPO_OWNER }} - tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '0.1.0' }} - app_version: ${{ steps.current_app_tag.outputs.latest_tag }} + tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '1.0.0' }} + app_version: ${{ steps.chart_version.outputs.app_version }} path: deploy/charts/${{ env.CHART_NAME }} registry: ${{ env.REGISTRY }} registry_username: ${{ github.actor }} diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index b77693ba8d5..b37597c7c82 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,13 +18,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.10 +version: 1.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: v1.50.2 +appVersion: v1.80.12 dependencies: - name: "postgresql" From 573df855d359c237d8d976d4708783e742cc8de4 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Mon, 12 Jan 2026 08:38:32 -0300 Subject: [PATCH 13/16] fix(oci): handle OpenAI-style image_url object in multimodal messages (#18272) The OCI adapter now accepts both string and object formats for image_url: - String: "image_url": "https://example.com/image.png" - Object: "image_url": {"url": "https://example.com/image.png"} This fixes compatibility with OpenAI Vision API format. --- litellm/llms/oci/chat/transformation.py | 5 +- .../oci/chat/test_oci_chat_transformation.py | 87 +++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 038895a39e5..7af7be2094a 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -1124,8 +1124,11 @@ def adapt_messages_to_generic_oci_standard_content_message( elif type == "image_url": image_url = content_item.get("image_url") + # Handle both OpenAI format (object with url) and string format + if isinstance(image_url, dict): + image_url = image_url.get("url") if not isinstance(image_url, str): - raise Exception("Prop `image_url` is not a string") + raise Exception("Prop `image_url` must be a string or an object with a `url` property") new_content.append(OCIImageContentPart(imageUrl=image_url)) return OCIMessage( diff --git a/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py index c23f4501a2c..f706a025a09 100644 --- a/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -197,3 +197,90 @@ class TestOCIGetCompleteUrl: assert "eu-frankfurt-1" in url assert "inference.generativeai" in url + + +class TestOCIImageUrlTransformation: + """Tests for OCI image_url format handling in multimodal messages. + + Fixes: https://github.com/BerriAI/litellm/issues/18270 + """ + + def test_image_url_as_string(self): + """Test that image_url as a plain string works.""" + from litellm.llms.oci.chat.transformation import adapt_messages_to_generic_oci_standard + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": "https://example.com/image.png"}, + ], + } + ] + + result = adapt_messages_to_generic_oci_standard(messages) + + assert len(result) == 1 + assert result[0].role == "USER" + assert len(result[0].content) == 2 + assert result[0].content[1].imageUrl == "https://example.com/image.png" + + def test_image_url_as_openai_object(self): + """Test that image_url as OpenAI-style object {"url": "..."} works.""" + from litellm.llms.oci.chat.transformation import adapt_messages_to_generic_oci_standard + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}, + ], + } + ] + + result = adapt_messages_to_generic_oci_standard(messages) + + assert len(result) == 1 + assert result[0].role == "USER" + assert len(result[0].content) == 2 + assert result[0].content[1].imageUrl == "https://example.com/image.png" + + def test_image_url_invalid_type_raises_error(self): + """Test that invalid image_url type raises an error.""" + from litellm.llms.oci.chat.transformation import adapt_messages_to_generic_oci_standard + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": 12345}, # Invalid type + ], + } + ] + + with pytest.raises(Exception) as exc_info: + adapt_messages_to_generic_oci_standard(messages) + + assert "image_url" in str(exc_info.value) + + def test_image_url_object_missing_url_raises_error(self): + """Test that object without 'url' property raises an error.""" + from litellm.llms.oci.chat.transformation import adapt_messages_to_generic_oci_standard + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"detail": "high"}}, # Missing 'url' + ], + } + ] + + with pytest.raises(Exception) as exc_info: + adapt_messages_to_generic_oci_standard(messages) + + assert "image_url" in str(exc_info.value) From 087ddee227b9c3d3f019ec51068027b0d6283367 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Mon, 12 Jan 2026 08:39:23 -0300 Subject: [PATCH 14/16] docs: update message content types link and add content types table (#18209) * docs: update message content types link and add content types table - Update "See All Message Values" link to point to main branch (line 664) instead of outdated commit 8600ec7 (line 392) - Add Content Types table documenting all 6 multimodal content types: text, image_url, input_audio, video_url, file, document - Link to existing docs for vision, audio, and document understanding * docs: add type definition links for text and video_url * docs: fix text type definition link to line 598 * docs: remove provider labels from file/document types * docs: add examples for all content types per review feedback --- docs/my-website/docs/completion/input.md | 42 +++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index 7df4f77017a..2f6da4bedcd 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -142,7 +142,47 @@ def completion( - `tool_call_id`: *str (optional)* - Tool call that this message is responding to. -[**See All Message Values**](https://github.com/BerriAI/litellm/blob/8600ec77042dacad324d3879a2bd918fc6a719fa/litellm/types/llms/openai.py#L392) +[**See All Message Values**](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L664) + +#### Content Types + +`content` can be a string (text only) or a list of content blocks (multimodal): + +| Type | Description | Docs | +|------|-------------|------| +| `text` | Text content | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L598) | +| `image_url` | Images | [Vision](./vision.md) | +| `input_audio` | Audio input | [Audio](./audio.md) | +| `video_url` | Video input | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L625) | +| `file` | Files | [Document Understanding](./document_understanding.md) | +| `document` | Documents/PDFs | [Document Understanding](./document_understanding.md) | + +**Examples:** +```python +# Text +messages=[{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}] + +# Image +messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}]}] + +# Audio +messages=[{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "", "format": "wav"}}]}] + +# Video +messages=[{"role": "user", "content": [{"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}]}] + +# File +messages=[{"role": "user", "content": [{"type": "file", "file": {"file_id": "https://example.com/doc.pdf"}}]}] + +# Document +messages=[{"role": "user", "content": [{"type": "document", "source": {"type": "text", "media_type": "application/pdf", "data": ""}}]}] + +# Combining multiple types (multimodal) +messages=[{"role": "user", "content": [ + {"type": "text", "text": "Generate a product description based on this image"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}} +]}] +``` ## Optional Fields From f7912990b76ee1305be2ae912493aa048edf1064 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Mon, 12 Jan 2026 08:42:19 -0300 Subject: [PATCH 15/16] fix(gemini): add presence_penalty support for Google AI Studio (#18154) Fixes #14753 Co-authored-by: Krish Dholakia --- litellm/llms/gemini/chat/transformation.py | 1 + .../test_vertex_and_google_ai_studio_gemini.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index 62897fe6ecb..f6d075392b2 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -87,6 +87,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): "stop", "logprobs", "frequency_penalty", + "presence_penalty", "modalities", "parallel_tool_calls", "web_search_options", diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 5d91275fa8d..af76a5edda7 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -14,6 +14,7 @@ from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) +from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.types.llms.vertex_ai import UsageMetadata from litellm.types.utils import ChoiceLogprobs, Usage from litellm.utils import CustomStreamWrapper @@ -2315,6 +2316,17 @@ def test_partial_json_chunk_on_first_chunk(): assert iterator.chunk_type == "accumulated_json", "Should switch to accumulated_json mode" + +def test_google_ai_studio_presence_penalty_supported(): + """ + Test that presence_penalty is supported for Google AI Studio Gemini. + + Regression test for https://github.com/BerriAI/litellm/issues/14753 + """ + config = GoogleAIStudioGeminiConfig() + supported_params = config.get_supported_openai_params(model="gemini-2.0-flash") + + assert "presence_penalty" in supported_params # ==================== Tool Type Separation Tests ==================== # These tests verify that each Tool object contains exactly one type per Vertex AI API spec # Ref: https://cloud.google.com/vertex-ai/generative-ai/docs/reference/rest/v1beta1/Tool From 9a8e781cb971a085e963d51061ad2e0d4ffa1f6b Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Mon, 12 Jan 2026 08:48:33 -0300 Subject: [PATCH 16/16] fix(anthropic): preserve web_fetch_tool_result in multi-turn conversations (#18142) Fixes #18137 Similar to the fix for web_search_tool_result (#17746, #17798), this PR preserves web_fetch_tool_result blocks in multi-turn conversations. Changes: - Add handling for web_fetch_tool_result in transformation.py (non-streaming) - Add capture of web_fetch_tool_result in handler.py (streaming) - Fix streaming tool arguments bug where empty input {} was prepended to actual arguments by using empty string instead of str({}) - Add unit tests for web_fetch_tool_result handling --- litellm/llms/anthropic/chat/handler.py | 13 ++ litellm/llms/anthropic/chat/transformation.py | 6 + .../chat/test_anthropic_chat_handler.py | 164 +++++++++++++++++- 3 files changed, 182 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 26e6016095e..2ad9a6cdf83 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -732,6 +732,19 @@ class ModelResponseIterator: provider_specific_fields["web_search_results"] = ( self.web_search_results ) + elif ( + content_block_start["content_block"]["type"] + == "web_fetch_tool_result" + ): + # Capture web_fetch_tool_result for multi-turn reconstruction + # The full content comes in content_block_start, not in deltas + # Fixes: https://github.com/BerriAI/litellm/issues/18137 + self.web_search_results.append( + content_block_start["content_block"] + ) + provider_specific_fields["web_search_results"] = ( + self.web_search_results + ) elif type_chunk == "content_block_stop": ContentBlockStop(**chunk) # type: ignore # check if tool call content block - only for tool_use and server_tool_use blocks diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 57391c152cb..6248f85223c 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1162,6 +1162,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if web_search_results is None: web_search_results = [] web_search_results.append(content) + ## WEB FETCH TOOL RESULT - preserve web fetch results for multi-turn conversations + ## Fixes: https://github.com/BerriAI/litellm/issues/18137 + elif content["type"] == "web_fetch_tool_result": + if web_search_results is None: + web_search_results = [] + web_search_results.append(content) elif content.get("thinking", None) is not None: if thinking_blocks is None: thinking_blocks = [] diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index 41febd4920a..d9f513d8d1d 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -473,7 +473,6 @@ def test_partial_json_chunk_accumulation(): streaming_response=MagicMock(), sync_stream=True, json_mode=False ) - # Simulate a complete JSON chunk being split into two parts partial_chunk_1 = '{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hel' partial_chunk_2 = 'lo"}}' @@ -781,6 +780,169 @@ def test_web_search_tool_result_captured_in_provider_specific_fields(): ), "First result title should match" +def test_web_fetch_tool_result_captured_in_provider_specific_fields(): + """ + Test that web_fetch_tool_result content is captured in provider_specific_fields. + + This tests the fix for https://github.com/BerriAI/litellm/issues/18137 + where streaming with Anthropic web fetch wasn't capturing web_fetch_tool_result + blocks, causing multi-turn conversations to fail. + + The web_fetch_tool_result content comes ALL AT ONCE in content_block_start, + not in deltas, so we need to capture it there. + """ + iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=False + ) + + # Simulate the streaming sequence with web_fetch_tool_result + chunks = [ + # 1. message_start + { + "type": "message_start", + "message": { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + }, + # 2. server_tool_use block starts (web_fetch) + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01ABC123", + "name": "web_fetch", + }, + }, + # 3. input_json_delta with the url + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com"}'}, + }, + # 4. content_block_stop for server_tool_use + {"type": "content_block_stop", "index": 0}, + # 5. web_fetch_tool_result block starts - THIS IS WHERE THE RESULTS ARE + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01ABC123", + "content": { + "type": "web_fetch_result", + "url": "https://example.com", + "retrieved_at": "2025-12-16T19:28:29.758000+00:00", + "content": { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "Hello World", + }, + "title": "Example Page", + }, + }, + }, + }, + # 6. content_block_stop for web_fetch_tool_result + {"type": "content_block_stop", "index": 1}, + ] + + web_search_results = None + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + if ( + parsed.choices + and parsed.choices[0].delta.provider_specific_fields + and "web_search_results" in parsed.choices[0].delta.provider_specific_fields + ): + web_search_results = parsed.choices[0].delta.provider_specific_fields[ + "web_search_results" + ] + + # Verify web_fetch_tool_result was captured (stored in web_search_results list) + assert web_search_results is not None, "web_search_results should be captured" + assert len(web_search_results) == 1, "Should have 1 web_fetch_tool_result block" + assert ( + web_search_results[0]["type"] == "web_fetch_tool_result" + ), "Block type should be web_fetch_tool_result" + assert ( + web_search_results[0]["tool_use_id"] == "srvtoolu_01ABC123" + ), "tool_use_id should match" + assert ( + web_search_results[0]["content"]["url"] == "https://example.com" + ), "URL should match" + assert ( + web_search_results[0]["content"]["content"]["title"] == "Example Page" + ), "Title should match" + + +def test_web_fetch_tool_result_no_extra_tool_calls(): + """ + Test that web_fetch_tool_result blocks don't emit tool call chunks. + + This tests the fix for https://github.com/BerriAI/litellm/issues/18137 + where streaming with Anthropic web fetch was causing issues with tool call arguments. + + The issue was that web_fetch_tool_result blocks have input_json_delta events with {} + that were incorrectly being converted to tool calls. + """ + iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=False + ) + + # to verify it doesn't emit tool calls + chunks = [ + # 1. web_fetch_tool_result block starts + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01ABC123", + "content": { + "type": "web_fetch_result", + "url": "https://example.com", + "retrieved_at": "2025-12-16T19:28:29.758000+00:00", + "content": { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "Hello World", + }, + "title": "Example Page", + }, + }, + }, + }, + # 2. input_json_delta with {} - this should NOT emit a tool call + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "input_json_delta", "partial_json": "{}"}, + }, + # 3. content_block_stop for web_fetch_tool_result + {"type": "content_block_stop", "index": 1}, + ] + + tool_call_count = 0 + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + if parsed.choices and parsed.choices[0].delta.tool_calls: + tool_call_count += 1 + + # Should have 0 tool calls - web_fetch_tool_result should not emit tool calls + assert ( + tool_call_count == 0 + ), f"Expected 0 tool calls, got {tool_call_count}. web_fetch_tool_result should not emit tool calls" + + def test_container_in_provider_specific_fields_streaming(): """ Test that container is captured in provider_specific_fields for streaming responses.