Merge pull request #19558 from BerriAI/litellm_gemini_vertexai_mapping

Add custom vertex ai mapping to the output
2026-09-12 23:01:41 +00:00 · 2026-01-22 19:44:32 +05:30 · 2026-01-22 19:44:32 +05:30 · c78c878822
commit c78c878822
parent 73715ab417 991fee056f
12 changed files with 238 additions and 20 deletions
--- a/litellm/constants.py
+++ b/litellm/constants.py
@ -1067,7 +1067,7 @@ known_tokenizer_config = {
 }


-OPENAI_FINISH_REASONS = ["stop", "length", "function_call", "content_filter", "null"]
+OPENAI_FINISH_REASONS = ["stop", "length", "function_call", "content_filter", "null", "finish_reason_unspecified", "malformed_function_call"]
 HUMANLOOP_PROMPT_CACHE_TTL_SECONDS = int(
    os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60)
 )  # 1 minute
--- a/litellm/google_genai/adapters/transformation.py
+++ b/litellm/google_genai/adapters/transformation.py
@ -2,7 +2,6 @@ import json
 from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union, cast

 from litellm import verbose_logger
-
 from litellm.litellm_core_utils.json_validation_rule import normalize_tool_schema
 from litellm.types.llms.openai import (
    AllMessageValues,
@ -771,6 +770,8 @@ class GoogleGenAIAdapter:
            "content_filter": "SAFETY",
            "tool_calls": "STOP",
            "function_call": "STOP",
+            "finish_reason_unspecified": "FINISH_REASON_UNSPECIFIED",
+            "malformed_function_call": "MALFORMED_FUNCTION_CALL",
        }

        return mapping.get(finish_reason, "STOP")
--- a/litellm/litellm_core_utils/core_helpers.py
+++ b/litellm/litellm_core_utils/core_helpers.py
@ -79,9 +79,11 @@ def map_finish_reason(
    elif finish_reason == "eos_token" or finish_reason == "stop_sequence":
        return "stop"
    elif (
-        finish_reason == "FINISH_REASON_UNSPECIFIED" or finish_reason == "STOP"
+        finish_reason == "FINISH_REASON_UNSPECIFIED"
    ):  # vertex ai - got from running `print(dir(response_obj.candidates[0].finish_reason))`: ['FINISH_REASON_UNSPECIFIED', 'MAX_TOKENS', 'OTHER', 'RECITATION', 'SAFETY', 'STOP',]
-        return "stop"
+        return "finish_reason_unspecified"
+    elif finish_reason == "MALFORMED_FUNCTION_CALL":
+        return "malformed_function_call"
    elif finish_reason == "SAFETY" or finish_reason == "RECITATION":  # vertex ai
        return "content_filter"
    elif finish_reason == "STOP":  # vertex ai
--- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
+++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
@ -1199,7 +1199,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
        and what it means
        """
        return {
-            "FINISH_REASON_UNSPECIFIED": "stop",  # openai doesn't have a way of representing this
+            "FINISH_REASON_UNSPECIFIED": "finish_reason_unspecified",
            "STOP": "stop",
            "MAX_TOKENS": "length",
            "SAFETY": "content_filter",
@ -1209,7 +1209,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
            "BLOCKLIST": "content_filter",
            "PROHIBITED_CONTENT": "content_filter",
            "SPII": "content_filter",
-            "MALFORMED_FUNCTION_CALL": "stop",  # openai doesn't have a way of representing this
+            "MALFORMED_FUNCTION_CALL": "malformed_function_call",  # openai doesn't have a way of representing this
            "IMAGE_SAFETY": "content_filter",
        }

--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@ -16863,14 +16863,14 @@
        "supports_vision": true
    },
    "gpt-4o-audio-preview": {
-        "input_cost_per_audio_token": 0.0001,
+        "input_cost_per_audio_token": 4e-05,
        "input_cost_per_token": 2.5e-06,
        "litellm_provider": "openai",
        "max_input_tokens": 128000,
        "max_output_tokens": 16384,
        "max_tokens": 16384,
        "mode": "chat",
-        "output_cost_per_audio_token": 0.0002,
+        "output_cost_per_audio_token": 8e-05,
        "output_cost_per_token": 1e-05,
        "supports_audio_input": true,
        "supports_audio_output": true,
@ -16880,14 +16880,14 @@
        "supports_tool_choice": true
    },
    "gpt-4o-audio-preview-2024-10-01": {
-        "input_cost_per_audio_token": 0.0001,
+        "input_cost_per_audio_token": 4e-05,
        "input_cost_per_token": 2.5e-06,
        "litellm_provider": "openai",
        "max_input_tokens": 128000,
        "max_output_tokens": 16384,
        "max_tokens": 16384,
        "mode": "chat",
-        "output_cost_per_audio_token": 0.0002,
+        "output_cost_per_audio_token": 8e-05,
        "output_cost_per_token": 1e-05,
        "supports_audio_input": true,
        "supports_audio_output": true,
@ -16930,6 +16930,186 @@
        "supports_system_messages": true,
        "supports_tool_choice": true
    },
+    "gpt-audio": {
+        "input_cost_per_audio_token": 3.2e-05,
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_audio_token": 6.4e-05,
+        "output_cost_per_token": 1e-05,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/realtime",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
+    "gpt-audio-2025-08-28": {
+        "input_cost_per_audio_token": 3.2e-05,
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_audio_token": 6.4e-05,
+        "output_cost_per_token": 1e-05,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/realtime",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
+    "gpt-audio-mini": {
+        "input_cost_per_audio_token": 1e-05,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_audio_token": 2e-05,
+        "output_cost_per_token": 2.4e-06,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/realtime",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
+    "gpt-audio-mini-2025-10-06": {
+        "input_cost_per_audio_token": 1e-05,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_audio_token": 2e-05,
+        "output_cost_per_token": 2.4e-06,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/realtime",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
+    "gpt-audio-mini-2025-12-15": {
+        "input_cost_per_audio_token": 1e-05,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_audio_token": 2e-05,
+        "output_cost_per_token": 2.4e-06,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/realtime",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
    "gpt-4o-mini": {
        "cache_read_input_token_cost": 7.5e-08,
        "cache_read_input_token_cost_priority": 1.25e-07,
--- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py
+++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py
@ -396,7 +396,7 @@ class AnthropicPassthroughLoggingHandler:
                
                # Add batch-specific metadata to indicate this is a pending batch job
                litellm_model_response.choices = [Choices(
-                    finish_reason="batch_pending",
+                    finish_reason="stop",
                    index=0,
                    message={
                        "role": "assistant",
@ -438,7 +438,7 @@ class AnthropicPassthroughLoggingHandler:
                
                # Add error-specific metadata
                litellm_model_response.choices = [Choices(
-                    finish_reason="batch_error",
+                    finish_reason="stop",
                    index=0,
                    message={
                        "role": "assistant",
@ -472,7 +472,7 @@ class AnthropicPassthroughLoggingHandler:
            
            # Add error-specific metadata
            litellm_model_response.choices = [Choices(
-                finish_reason="batch_error",
+                finish_reason="stop",
                index=0,
                message={
                    "role": "assistant",
--- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
+++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
@ -619,7 +619,7 @@ class VertexPassthroughLoggingHandler:
                
                # Add batch-specific metadata to indicate this is a pending batch job
                litellm_model_response.choices = [Choices(
-                    finish_reason="batch_pending",
+                    finish_reason="stop",
                    index=0,
                    message={
                        "role": "assistant",
@ -661,7 +661,7 @@ class VertexPassthroughLoggingHandler:
                
                # Add error-specific metadata
                litellm_model_response.choices = [Choices(
-                    finish_reason="batch_error",
+                    finish_reason="stop",
                    index=0,
                    message={
                        "role": "assistant",
@ -695,7 +695,7 @@ class VertexPassthroughLoggingHandler:
            
            # Add error-specific metadata
            litellm_model_response.choices = [Choices(
-                finish_reason="batch_error",
+                finish_reason="stop",
                index=0,
                message={
                    "role": "assistant",
--- a/litellm/types/llms/openai.py
+++ b/litellm/types/llms/openai.py
@ -2012,7 +2012,7 @@ class OpenAIBatchResult(TypedDict, total=False):


 OpenAIChatCompletionFinishReason = Literal[
-    "stop", "content_filter", "function_call", "tool_calls", "length"
+    "stop", "content_filter", "function_call", "tool_calls", "length", "finish_reason_unspecified", "malformed_function_call" # last 2 are vertex ai specific
 ]


--- a/litellm/types/llms/vertex_ai.py
+++ b/litellm/types/llms/vertex_ai.py
@ -396,6 +396,8 @@ class Candidates(TypedDict, total=False):
        "BLOCKLIST",
        "PROHIBITED_CONTENT",
        "SPII",
+        "MALFORMED_FUNCTION_CALL",
+        "IMAGE_SAFETY",
    ]
    safetyRatings: List[SafetyRatings]
    citationMetadata: CitationMetadata
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@ -46,6 +46,7 @@ from .llms.openai import (
    FineTuningJob,
    ImageURLListItem,
    OpenAIChatCompletionChunk,
+    OpenAIChatCompletionFinishReason,
    OpenAIFileObject,
    OpenAIRealtimeStreamList,
    ResponsesAPIResponse,
@ -1254,7 +1255,7 @@ class Delta(SafeAttributeModel, OpenAIObject):


 class Choices(SafeAttributeModel, OpenAIObject):
-    finish_reason: str
+    finish_reason: OpenAIChatCompletionFinishReason
    index: int
    message: Message
    logprobs: Optional[Union[ChoiceLogprobs, Any]] = None
--- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
+++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
@ -10,11 +10,11 @@ from pydantic import BaseModel

 import litellm
 from litellm import ModelResponse, completion
+from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig
 from litellm.llms.vertex_ai.common_utils import VertexAIError
 from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
    VertexGeminiConfig,
 )
-from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig
 from litellm.types.llms.vertex_ai import UsageMetadata
 from litellm.types.utils import ChoiceLogprobs, Usage
 from litellm.utils import CustomStreamWrapper
@ -606,6 +606,38 @@ def test_check_finish_reason():
        )


+def test_finish_reason_unspecified_and_malformed_function_call():
+    """
+    Test that FINISH_REASON_UNSPECIFIED and MALFORMED_FUNCTION_CALL 
+    return their lowercase values instead of being mapped to 'stop'
+    since we don't have good mappings for these.
+    """
+    finish_reason_mappings = VertexGeminiConfig.get_finish_reason_mapping()
+    
+    # Test FINISH_REASON_UNSPECIFIED returns lowercase version
+    assert finish_reason_mappings["FINISH_REASON_UNSPECIFIED"] == "finish_reason_unspecified"
+    assert (
+        VertexGeminiConfig._check_finish_reason(
+            chat_completion_message=None, finish_reason="FINISH_REASON_UNSPECIFIED"
+        )
+        == "finish_reason_unspecified"
+    )
+    
+    # Test MALFORMED_FUNCTION_CALL returns lowercase version
+    assert finish_reason_mappings["MALFORMED_FUNCTION_CALL"] == "malformed_function_call"
+    assert (
+        VertexGeminiConfig._check_finish_reason(
+            chat_completion_message=None, finish_reason="MALFORMED_FUNCTION_CALL"
+        )
+        == "malformed_function_call"
+    )
+    
+    # Ensure these values are in the OpenAI finish reasons constant
+    from litellm import OPENAI_FINISH_REASONS
+    assert "finish_reason_unspecified" in OPENAI_FINISH_REASONS
+    assert "malformed_function_call" in OPENAI_FINISH_REASONS
+
+
 def test_vertex_ai_usage_metadata_response_token_count():
    """For Gemini Live API"""
    from litellm.types.utils import PromptTokensDetailsWrapper
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py
@ -127,7 +127,7 @@ class TestVertexAIBatchPassthroughHandler:
            assert result is not None
            assert "result" in result
            assert "kwargs" in result
-            assert result["result"].choices[0].finish_reason == "batch_error"
+            assert result["result"].choices[0].finish_reason == "stop"
            assert result["kwargs"]["batch_job_state"] == "JOB_STATE_FAILED"

    def test_get_actual_model_id_from_router_with_router(self):