diff --git a/litellm/constants.py b/litellm/constants.py index 5da0cf1489f..d346bc46f51 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1067,7 +1067,7 @@ known_tokenizer_config = { } -OPENAI_FINISH_REASONS = ["stop", "length", "function_call", "content_filter", "null"] +OPENAI_FINISH_REASONS = ["stop", "length", "function_call", "content_filter", "null", "finish_reason_unspecified", "malformed_function_call"] HUMANLOOP_PROMPT_CACHE_TTL_SECONDS = int( os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) ) # 1 minute diff --git a/litellm/google_genai/adapters/transformation.py b/litellm/google_genai/adapters/transformation.py index 58a52666d38..0a296012210 100644 --- a/litellm/google_genai/adapters/transformation.py +++ b/litellm/google_genai/adapters/transformation.py @@ -2,7 +2,6 @@ import json from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union, cast from litellm import verbose_logger - from litellm.litellm_core_utils.json_validation_rule import normalize_tool_schema from litellm.types.llms.openai import ( AllMessageValues, @@ -771,6 +770,8 @@ class GoogleGenAIAdapter: "content_filter": "SAFETY", "tool_calls": "STOP", "function_call": "STOP", + "finish_reason_unspecified": "FINISH_REASON_UNSPECIFIED", + "malformed_function_call": "MALFORMED_FUNCTION_CALL", } return mapping.get(finish_reason, "STOP") diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index dadb36f3fd7..9cb0a00d9fc 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -79,9 +79,11 @@ def map_finish_reason( elif finish_reason == "eos_token" or finish_reason == "stop_sequence": return "stop" elif ( - finish_reason == "FINISH_REASON_UNSPECIFIED" or finish_reason == "STOP" + finish_reason == "FINISH_REASON_UNSPECIFIED" ): # vertex ai - got from running `print(dir(response_obj.candidates[0].finish_reason))`: ['FINISH_REASON_UNSPECIFIED', 'MAX_TOKENS', 'OTHER', 'RECITATION', 'SAFETY', 'STOP',] - return "stop" + return "finish_reason_unspecified" + elif finish_reason == "MALFORMED_FUNCTION_CALL": + return "malformed_function_call" elif finish_reason == "SAFETY" or finish_reason == "RECITATION": # vertex ai return "content_filter" elif finish_reason == "STOP": # vertex ai diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 2d2e07e74db..b78ac8f9e98 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1199,7 +1199,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): and what it means """ return { - "FINISH_REASON_UNSPECIFIED": "stop", # openai doesn't have a way of representing this + "FINISH_REASON_UNSPECIFIED": "finish_reason_unspecified", "STOP": "stop", "MAX_TOKENS": "length", "SAFETY": "content_filter", @@ -1209,7 +1209,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "BLOCKLIST": "content_filter", "PROHIBITED_CONTENT": "content_filter", "SPII": "content_filter", - "MALFORMED_FUNCTION_CALL": "stop", # openai doesn't have a way of representing this + "MALFORMED_FUNCTION_CALL": "malformed_function_call", # openai doesn't have a way of representing this "IMAGE_SAFETY": "content_filter", } diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a74b80e7373..ab034d9f51b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16863,14 +16863,14 @@ "supports_vision": true }, "gpt-4o-audio-preview": { - "input_cost_per_audio_token": 0.0001, + "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "output_cost_per_audio_token": 0.0002, + "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, "supports_audio_input": true, "supports_audio_output": true, @@ -16880,14 +16880,14 @@ "supports_tool_choice": true }, "gpt-4o-audio-preview-2024-10-01": { - "input_cost_per_audio_token": 0.0001, + "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "output_cost_per_audio_token": 0.0002, + "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, "supports_audio_input": true, "supports_audio_output": true, @@ -16930,6 +16930,186 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "gpt-audio": { + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/realtime", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "gpt-audio-2025-08-28": { + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/realtime", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "gpt-audio-mini": { + "input_cost_per_audio_token": 1e-05, + "input_cost_per_token": 6e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/realtime", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "gpt-audio-mini-2025-10-06": { + "input_cost_per_audio_token": 1e-05, + "input_cost_per_token": 6e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/realtime", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "gpt-audio-mini-2025-12-15": { + "input_cost_per_audio_token": 1e-05, + "input_cost_per_token": 6e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/realtime", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "gpt-4o-mini": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.25e-07, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 4e1112329ee..e70d6cb7fca 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -396,7 +396,7 @@ class AnthropicPassthroughLoggingHandler: # Add batch-specific metadata to indicate this is a pending batch job litellm_model_response.choices = [Choices( - finish_reason="batch_pending", + finish_reason="stop", index=0, message={ "role": "assistant", @@ -438,7 +438,7 @@ class AnthropicPassthroughLoggingHandler: # Add error-specific metadata litellm_model_response.choices = [Choices( - finish_reason="batch_error", + finish_reason="stop", index=0, message={ "role": "assistant", @@ -472,7 +472,7 @@ class AnthropicPassthroughLoggingHandler: # Add error-specific metadata litellm_model_response.choices = [Choices( - finish_reason="batch_error", + finish_reason="stop", index=0, message={ "role": "assistant", diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 0962fafe3f6..3d5c529a3bb 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -619,7 +619,7 @@ class VertexPassthroughLoggingHandler: # Add batch-specific metadata to indicate this is a pending batch job litellm_model_response.choices = [Choices( - finish_reason="batch_pending", + finish_reason="stop", index=0, message={ "role": "assistant", @@ -661,7 +661,7 @@ class VertexPassthroughLoggingHandler: # Add error-specific metadata litellm_model_response.choices = [Choices( - finish_reason="batch_error", + finish_reason="stop", index=0, message={ "role": "assistant", @@ -695,7 +695,7 @@ class VertexPassthroughLoggingHandler: # Add error-specific metadata litellm_model_response.choices = [Choices( - finish_reason="batch_error", + finish_reason="stop", index=0, message={ "role": "assistant", diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 59835b05184..879f8ba800d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -2012,7 +2012,7 @@ class OpenAIBatchResult(TypedDict, total=False): OpenAIChatCompletionFinishReason = Literal[ - "stop", "content_filter", "function_call", "tool_calls", "length" + "stop", "content_filter", "function_call", "tool_calls", "length", "finish_reason_unspecified", "malformed_function_call" # last 2 are vertex ai specific ] diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 0a4a2d0f14c..049a5010c79 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -396,6 +396,8 @@ class Candidates(TypedDict, total=False): "BLOCKLIST", "PROHIBITED_CONTENT", "SPII", + "MALFORMED_FUNCTION_CALL", + "IMAGE_SAFETY", ] safetyRatings: List[SafetyRatings] citationMetadata: CitationMetadata diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 41a09bd3917..cd797dd1e54 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -46,6 +46,7 @@ from .llms.openai import ( FineTuningJob, ImageURLListItem, OpenAIChatCompletionChunk, + OpenAIChatCompletionFinishReason, OpenAIFileObject, OpenAIRealtimeStreamList, ResponsesAPIResponse, @@ -1254,7 +1255,7 @@ class Delta(SafeAttributeModel, OpenAIObject): class Choices(SafeAttributeModel, OpenAIObject): - finish_reason: str + finish_reason: OpenAIChatCompletionFinishReason index: int message: Message logprobs: Optional[Union[ChoiceLogprobs, Any]] = None diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 5be080b53fa..ac099a0168c 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -10,11 +10,11 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse, completion +from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) -from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.types.llms.vertex_ai import UsageMetadata from litellm.types.utils import ChoiceLogprobs, Usage from litellm.utils import CustomStreamWrapper @@ -606,6 +606,38 @@ def test_check_finish_reason(): ) +def test_finish_reason_unspecified_and_malformed_function_call(): + """ + Test that FINISH_REASON_UNSPECIFIED and MALFORMED_FUNCTION_CALL + return their lowercase values instead of being mapped to 'stop' + since we don't have good mappings for these. + """ + finish_reason_mappings = VertexGeminiConfig.get_finish_reason_mapping() + + # Test FINISH_REASON_UNSPECIFIED returns lowercase version + assert finish_reason_mappings["FINISH_REASON_UNSPECIFIED"] == "finish_reason_unspecified" + assert ( + VertexGeminiConfig._check_finish_reason( + chat_completion_message=None, finish_reason="FINISH_REASON_UNSPECIFIED" + ) + == "finish_reason_unspecified" + ) + + # Test MALFORMED_FUNCTION_CALL returns lowercase version + assert finish_reason_mappings["MALFORMED_FUNCTION_CALL"] == "malformed_function_call" + assert ( + VertexGeminiConfig._check_finish_reason( + chat_completion_message=None, finish_reason="MALFORMED_FUNCTION_CALL" + ) + == "malformed_function_call" + ) + + # Ensure these values are in the OpenAI finish reasons constant + from litellm import OPENAI_FINISH_REASONS + assert "finish_reason_unspecified" in OPENAI_FINISH_REASONS + assert "malformed_function_call" in OPENAI_FINISH_REASONS + + def test_vertex_ai_usage_metadata_response_token_count(): """For Gemini Live API""" from litellm.types.utils import PromptTokensDetailsWrapper diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py index 68c6bf98cb2..66c063d47d8 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py @@ -127,7 +127,7 @@ class TestVertexAIBatchPassthroughHandler: assert result is not None assert "result" in result assert "kwargs" in result - assert result["result"].choices[0].finish_reason == "batch_error" + assert result["result"].choices[0].finish_reason == "stop" assert result["kwargs"]["batch_job_state"] == "JOB_STATE_FAILED" def test_get_actual_model_id_from_router_with_router(self):