diff --git a/docs/my-website/docs/completion/output.md b/docs/my-website/docs/completion/output.md index f705bc9f311..a7f26a0ec37 100644 --- a/docs/my-website/docs/completion/output.md +++ b/docs/my-website/docs/completion/output.md @@ -51,6 +51,28 @@ Here's what an example response looks like } ``` +## Native Finish Reason + +LiteLLM maps all provider-specific `finish_reason` values to OpenAI-compatible values (`stop`, `length`, `tool_calls`, `function_call`, `content_filter`). When the original provider value differs from the mapped value, it is preserved in `provider_specific_fields["native_finish_reason"]`. + +This is useful for agent loops that need to distinguish between different stop conditions (e.g., Gemini's `MALFORMED_FUNCTION_CALL` vs a normal `stop`). + +```python +response = completion(model="gemini/gemini-2.0-flash", messages=messages) + +choice = response.choices[0] +print(choice.finish_reason) # "stop" (OpenAI-compatible) + +# Access the original provider value when it differs: +if hasattr(choice, "provider_specific_fields") and choice.provider_specific_fields: + native = choice.provider_specific_fields.get("native_finish_reason") + if native == "MALFORMED_FUNCTION_CALL": + # Handle malformed function call differently from a normal stop + pass +``` + +When the provider already returns an OpenAI-compatible value (e.g., `stop`), `native_finish_reason` is not set. + ## Additional Attributes You can also access information like latency. diff --git a/litellm/constants.py b/litellm/constants.py index 2ae365300ef..2486c223ec1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1214,12 +1214,8 @@ OPENAI_FINISH_REASONS = [ "stop", "length", "function_call", + "tool_calls", "content_filter", - "null", - "finish_reason_unspecified", - "malformed_function_call", - "guardrail_intervened", - "eos", ] HUMANLOOP_PROMPT_CACHE_TTL_SECONDS = int( os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) diff --git a/litellm/google_genai/adapters/transformation.py b/litellm/google_genai/adapters/transformation.py index 0a296012210..c5d9fd124fa 100644 --- a/litellm/google_genai/adapters/transformation.py +++ b/litellm/google_genai/adapters/transformation.py @@ -770,8 +770,6 @@ class GoogleGenAIAdapter: "content_filter": "SAFETY", "tool_calls": "STOP", "function_call": "STOP", - "finish_reason_unspecified": "FINISH_REASON_UNSPECIFIED", - "malformed_function_call": "MALFORMED_FUNCTION_CALL", } return mapping.get(finish_reason, "STOP") diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 4113fa351ae..85ed955af4a 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -58,63 +58,55 @@ def safe_divide( return numerator / denominator -# Module-level constant derived from the source-of-truth Literal type. -# Avoids recreating the set on every call (map_finish_reason is called per-chunk -# during streaming) and stays in sync when the Literal is updated. -_VALID_OPENAI_FINISH_REASONS = frozenset(get_args(OpenAIChatCompletionFinishReason)) +_FINISH_REASON_MAP: dict[str, OpenAIChatCompletionFinishReason] = { + # Anthropic + "stop_sequence": "stop", + "end_turn": "stop", + "max_tokens": "length", + "tool_use": "tool_calls", + "compaction": "length", + # Cohere + "COMPLETE": "stop", + "ERROR_TOXIC": "content_filter", + "ERROR": "stop", + # HuggingFace / Together AI + "eos_token": "stop", + "eos": "stop", + # Gemini / Vertex AI + "STOP": "stop", + "MAX_TOKENS": "length", + "SAFETY": "content_filter", + "RECITATION": "content_filter", + "FINISH_REASON_UNSPECIFIED": "stop", + "MALFORMED_FUNCTION_CALL": "stop", + "LANGUAGE": "content_filter", + "OTHER": "content_filter", + "BLOCKLIST": "content_filter", + "PROHIBITED_CONTENT": "content_filter", + "SPII": "content_filter", + "IMAGE_SAFETY": "content_filter", + "IMAGE_PROHIBITED_CONTENT": "content_filter", + "TOO_MANY_TOOL_CALLS": "stop", + "MALFORMED_RESPONSE": "stop", + # Bedrock + "guardrail_intervened": "content_filter", + # OpenAI passthrough + "stop": "stop", + "length": "length", + "tool_calls": "tool_calls", + "function_call": "function_call", + "content_filter": "content_filter", +} -def map_finish_reason( - finish_reason: str, -): # openai supports 5 stop sequences - 'stop', 'length', 'function_call', 'content_filter', 'null' - # anthropic mapping - if finish_reason == "stop_sequence": - return "stop" - # cohere mapping - https://docs.cohere.com/reference/generate - elif finish_reason == "COMPLETE": - return "stop" - elif finish_reason == "MAX_TOKENS": # cohere + vertex ai - return "length" - elif finish_reason == "ERROR_TOXIC": - return "content_filter" - elif ( - finish_reason == "ERROR" - ): # openai currently doesn't support an 'error' finish reason - return "stop" - # huggingface mapping https://huggingface.github.io/text-generation-inference/#/Text%20Generation%20Inference/generate_stream - elif finish_reason == "eos_token" or finish_reason == "stop_sequence": - return "stop" - elif ( - finish_reason == "FINISH_REASON_UNSPECIFIED" - ): # vertex ai - got from running `print(dir(response_obj.candidates[0].finish_reason))`: ['FINISH_REASON_UNSPECIFIED', 'MAX_TOKENS', 'OTHER', 'RECITATION', 'SAFETY', 'STOP',] - return "finish_reason_unspecified" - elif finish_reason == "MALFORMED_FUNCTION_CALL": - return "malformed_function_call" - elif finish_reason == "SAFETY" or finish_reason == "RECITATION": # vertex ai - return "content_filter" - elif finish_reason == "STOP": # vertex ai - return "stop" - elif finish_reason == "end_turn" or finish_reason == "stop_sequence": # anthropic - return "stop" - elif finish_reason == "max_tokens": # anthropic - return "length" - elif finish_reason == "tool_use": # anthropic - return "tool_calls" - elif finish_reason == "compaction": - return "length" - # Unknown finish_reason values (e.g. provider-specific error codes like - # "network_error" from ZhipuAI/GLM-5) are not in OpenAIChatCompletionFinishReason - # Literal and will cause a Pydantic ValidationError in Choices.__init__. - # Map them to "finish_reason_unspecified" so the stream can be assembled - # without raising an exception. - if finish_reason not in _VALID_OPENAI_FINISH_REASONS: +def map_finish_reason(finish_reason: str) -> OpenAIChatCompletionFinishReason: + mapped = _FINISH_REASON_MAP.get(finish_reason) + if mapped is None: verbose_logger.warning( - "litellm.map_finish_reason: unknown finish_reason %r from provider; " - "mapping to 'finish_reason_unspecified' to avoid ValidationError.", - finish_reason, + "Unmapped finish_reason '%s', defaulting to 'stop'", finish_reason ) - return "finish_reason_unspecified" - return finish_reason + return "stop" + return mapped def remove_index_from_tool_calls( diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 9b3a94904a7..0c26e9ca5ed 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1242,27 +1242,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "IMAGE_PROHIBITED_CONTENT": "The token generation was stopped as the response was flagged for prohibited image content.", } + _GEMINI_FINISH_REASON_KEYS = frozenset({ + "STOP", "MAX_TOKENS", "SAFETY", "RECITATION", "FINISH_REASON_UNSPECIFIED", + "MALFORMED_FUNCTION_CALL", "LANGUAGE", "OTHER", "BLOCKLIST", + "PROHIBITED_CONTENT", "SPII", "IMAGE_SAFETY", "IMAGE_PROHIBITED_CONTENT", + "TOO_MANY_TOOL_CALLS", "MALFORMED_RESPONSE", + }) + @staticmethod def get_finish_reason_mapping() -> Dict[str, OpenAIChatCompletionFinishReason]: """ - Return Dictionary of finish reasons which indicate response was flagged - - and what it means + Return Dictionary of Gemini/Vertex AI finish reasons and their + OpenAI-compatible mappings. """ + from litellm.litellm_core_utils.core_helpers import _FINISH_REASON_MAP + return { - "FINISH_REASON_UNSPECIFIED": "finish_reason_unspecified", - "STOP": "stop", - "MAX_TOKENS": "length", - "SAFETY": "content_filter", - "RECITATION": "content_filter", - "LANGUAGE": "content_filter", - "OTHER": "content_filter", - "BLOCKLIST": "content_filter", - "PROHIBITED_CONTENT": "content_filter", - "SPII": "content_filter", - "MALFORMED_FUNCTION_CALL": "malformed_function_call", # openai doesn't have a way of representing this - "IMAGE_SAFETY": "content_filter", - "IMAGE_PROHIBITED_CONTENT": "content_filter", + k: v + for k, v in _FINISH_REASON_MAP.items() + if k in VertexGeminiConfig._GEMINI_FINISH_REASON_KEYS } def translate_exception_str(self, exception_string: str): @@ -1781,15 +1779,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_message: Optional[ChatCompletionResponseMessage], finish_reason: Optional[str], ) -> OpenAIChatCompletionFinishReason: - mapped_finish_reason = VertexGeminiConfig.get_finish_reason_mapping() + from litellm.litellm_core_utils.core_helpers import map_finish_reason + if chat_completion_message and chat_completion_message.get("function_call"): return "function_call" elif chat_completion_message and chat_completion_message.get("tool_calls"): return "tool_calls" - elif ( - finish_reason and finish_reason in mapped_finish_reason.keys() - ): # vertex ai - return mapped_finish_reason[finish_reason] + elif finish_reason: + return map_finish_reason(finish_reason) else: return "stop" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 8a2c5efc268..8b81bcf2a44 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -2110,7 +2110,7 @@ class OpenAIBatchResult(TypedDict, total=False): OpenAIChatCompletionFinishReason = Literal[ - "stop", "content_filter", "function_call", "tool_calls", "length", "guardrail_intervened", "eos", "finish_reason_unspecified", "malformed_function_call" # last 2 are vertex ai specific, guardrail_intervened is bedrock specific + "stop", "content_filter", "function_call", "tool_calls", "length" ] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b5d5c06924d..ecf4669f3d0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1326,7 +1326,11 @@ class Choices(SafeAttributeModel, OpenAIObject): **params, ): if finish_reason is not None: - params["finish_reason"] = map_finish_reason(finish_reason) + mapped = map_finish_reason(finish_reason) + params["finish_reason"] = mapped + if finish_reason != mapped: + provider_specific_fields = dict(provider_specific_fields) if provider_specific_fields else {} + provider_specific_fields["native_finish_reason"] = finish_reason else: params["finish_reason"] = "stop" if index is not None: diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py index 4118b4cb058..0ef76e0942d 100644 --- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py @@ -2,65 +2,11 @@ import pytest -from litellm.litellm_core_utils.core_helpers import map_finish_reason, reconstruct_model_name - - -class TestMapFinishReason: - @pytest.mark.parametrize( - "value", - [ - "stop", - "length", - "function_call", - "tool_calls", - "content_filter", - "finish_reason_unspecified", - "eos", - "guardrail_intervened", - "malformed_function_call", - ], - ) - def test_known_openai_values_pass_through(self, value: str) -> None: - assert map_finish_reason(value) == value - - def test_anthropic_tool_use_maps_to_tool_calls(self) -> None: - assert map_finish_reason("tool_use") == "tool_calls" - - def test_anthropic_max_tokens_maps_to_length(self) -> None: - assert map_finish_reason("max_tokens") == "length" - - def test_anthropic_end_turn_maps_to_stop(self) -> None: - assert map_finish_reason("end_turn") == "stop" - - def test_cohere_complete_maps_to_stop(self) -> None: - assert map_finish_reason("COMPLETE") == "stop" - - def test_cohere_max_tokens_maps_to_length(self) -> None: - assert map_finish_reason("MAX_TOKENS") == "length" - - def test_cohere_error_toxic_maps_to_content_filter(self) -> None: - assert map_finish_reason("ERROR_TOXIC") == "content_filter" - - def test_vertex_ai_stop_maps_to_stop(self) -> None: - assert map_finish_reason("STOP") == "stop" - - def test_vertex_ai_safety_maps_to_content_filter(self) -> None: - assert map_finish_reason("SAFETY") == "content_filter" - - def test_vertex_ai_finish_reason_unspecified_maps_correctly(self) -> None: - assert map_finish_reason("FINISH_REASON_UNSPECIFIED") == "finish_reason_unspecified" - - def test_vertex_ai_malformed_function_call_maps_correctly(self) -> None: - assert map_finish_reason("MALFORMED_FUNCTION_CALL") == "malformed_function_call" - - def test_unknown_value_maps_to_finish_reason_unspecified(self) -> None: - assert map_finish_reason("some_unknown_reason") == "finish_reason_unspecified" - - def test_empty_string_maps_to_finish_reason_unspecified(self) -> None: - assert map_finish_reason("") == "finish_reason_unspecified" - - def test_zhipuai_glm_network_error_regression(self) -> None: - assert map_finish_reason("network_error") == "finish_reason_unspecified" +from litellm.litellm_core_utils.core_helpers import ( + _FINISH_REASON_MAP, + map_finish_reason, + reconstruct_model_name, +) def test_reconstruct_model_name_prefers_deployment_value(): @@ -103,3 +49,102 @@ def test_reconstruct_model_name_returns_original_for_other_providers(): ) assert result == "claude-3-sonnet" + + +# --------------------------------------------------------------------------- +# map_finish_reason tests +# --------------------------------------------------------------------------- + +VALID_OPENAI_FINISH_REASONS = {"stop", "length", "tool_calls", "function_call", "content_filter"} + + +class TestMapFinishReasonAnthropic: + def test_stop_sequence(self): + assert map_finish_reason("stop_sequence") == "stop" + + def test_end_turn(self): + assert map_finish_reason("end_turn") == "stop" + + def test_max_tokens(self): + assert map_finish_reason("max_tokens") == "length" + + def test_tool_use(self): + assert map_finish_reason("tool_use") == "tool_calls" + + def test_compaction(self): + assert map_finish_reason("compaction") == "length" + + +class TestMapFinishReasonGemini: + @pytest.mark.parametrize( + "gemini_reason,expected", + [ + ("STOP", "stop"), + ("MAX_TOKENS", "length"), + ("SAFETY", "content_filter"), + ("RECITATION", "content_filter"), + ("FINISH_REASON_UNSPECIFIED", "stop"), + ("MALFORMED_FUNCTION_CALL", "stop"), + ("LANGUAGE", "content_filter"), + ("OTHER", "content_filter"), + ("BLOCKLIST", "content_filter"), + ("PROHIBITED_CONTENT", "content_filter"), + ("SPII", "content_filter"), + ("IMAGE_SAFETY", "content_filter"), + ("IMAGE_PROHIBITED_CONTENT", "content_filter"), + ("TOO_MANY_TOOL_CALLS", "stop"), + ("MALFORMED_RESPONSE", "stop"), + ], + ) + def test_gemini_finish_reasons(self, gemini_reason, expected): + assert map_finish_reason(gemini_reason) == expected + + +class TestMapFinishReasonCohere: + def test_complete(self): + assert map_finish_reason("COMPLETE") == "stop" + + def test_error_toxic(self): + assert map_finish_reason("ERROR_TOXIC") == "content_filter" + + def test_error(self): + assert map_finish_reason("ERROR") == "stop" + + +class TestMapFinishReasonHuggingFace: + def test_eos_token(self): + assert map_finish_reason("eos_token") == "stop" + + def test_eos(self): + assert map_finish_reason("eos") == "stop" + + +class TestMapFinishReasonBedrock: + def test_guardrail_intervened(self): + assert map_finish_reason("guardrail_intervened") == "content_filter" + + +class TestMapFinishReasonOpenAIPassthrough: + @pytest.mark.parametrize( + "reason", ["stop", "length", "tool_calls", "function_call", "content_filter"] + ) + def test_openai_values_pass_through(self, reason): + assert map_finish_reason(reason) == reason + + +class TestMapFinishReasonUnknown: + def test_unknown_value_defaults_to_stop(self): + assert map_finish_reason("some_unknown_value") == "stop" + + def test_empty_string_defaults_to_stop(self): + assert map_finish_reason("") == "stop" + + +class TestFinishReasonMapOutputsAreValid: + def test_all_mapped_values_are_valid_openai_reasons(self): + """Every value in _FINISH_REASON_MAP must be a valid OpenAI finish reason.""" + for provider_reason, openai_reason in _FINISH_REASON_MAP.items(): + assert openai_reason in VALID_OPENAI_FINISH_REASONS, ( + f"Mapped value '{openai_reason}' (from '{provider_reason}') " + f"is not a valid OpenAI finish reason" + ) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 8beb19bf1ac..38be5de42b0 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -674,34 +674,32 @@ def test_check_finish_reason(): def test_finish_reason_unspecified_and_malformed_function_call(): """ - Test that FINISH_REASON_UNSPECIFIED and MALFORMED_FUNCTION_CALL - return their lowercase values instead of being mapped to 'stop' - since we don't have good mappings for these. + Test that FINISH_REASON_UNSPECIFIED and MALFORMED_FUNCTION_CALL + are mapped to OpenAI-compatible 'stop' finish reason. """ finish_reason_mappings = VertexGeminiConfig.get_finish_reason_mapping() - - # Test FINISH_REASON_UNSPECIFIED returns lowercase version - assert finish_reason_mappings["FINISH_REASON_UNSPECIFIED"] == "finish_reason_unspecified" + + # Test FINISH_REASON_UNSPECIFIED maps to "stop" + assert finish_reason_mappings["FINISH_REASON_UNSPECIFIED"] == "stop" assert ( VertexGeminiConfig._check_finish_reason( chat_completion_message=None, finish_reason="FINISH_REASON_UNSPECIFIED" ) - == "finish_reason_unspecified" + == "stop" ) - - # Test MALFORMED_FUNCTION_CALL returns lowercase version - assert finish_reason_mappings["MALFORMED_FUNCTION_CALL"] == "malformed_function_call" + + # Test MALFORMED_FUNCTION_CALL maps to "stop" + assert finish_reason_mappings["MALFORMED_FUNCTION_CALL"] == "stop" assert ( VertexGeminiConfig._check_finish_reason( chat_completion_message=None, finish_reason="MALFORMED_FUNCTION_CALL" ) - == "malformed_function_call" + == "stop" ) - - # Ensure these values are in the OpenAI finish reasons constant - from litellm import OPENAI_FINISH_REASONS - assert "finish_reason_unspecified" in OPENAI_FINISH_REASONS - assert "malformed_function_call" in OPENAI_FINISH_REASONS + + # Test new Gemini finish reasons + assert finish_reason_mappings["TOO_MANY_TOOL_CALLS"] == "stop" + assert finish_reason_mappings["MALFORMED_RESPONSE"] == "stop" def test_vertex_ai_usage_metadata_response_token_count(): diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index 8c20ace98a0..ccd467a1f7b 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -223,3 +223,59 @@ def test_chat_completion_token_logprob_invalid_top_logprobs_rejected(): logprob=-0.31725305, top_logprobs="invalid_string", ) + + +# --------------------------------------------------------------------------- +# native_finish_reason in provider_specific_fields +# --------------------------------------------------------------------------- + + +class TestNativeFinishReason: + """Choices exposes the raw provider finish_reason in provider_specific_fields + when it differs from the mapped OpenAI-compatible value.""" + + def test_provider_reason_exposed_when_mapped(self): + from litellm.types.utils import Choices + + choice = Choices(finish_reason="end_turn") + assert choice.finish_reason == "stop" + assert choice.provider_specific_fields["native_finish_reason"] == "end_turn" + + def test_provider_reason_not_set_when_already_openai(self): + from litellm.types.utils import Choices + + choice = Choices(finish_reason="stop") + assert choice.finish_reason == "stop" + assert not hasattr(choice, "provider_specific_fields") + + def test_provider_reason_merged_with_existing_fields(self): + from litellm.types.utils import Choices + + choice = Choices( + finish_reason="max_tokens", + provider_specific_fields={"citations": [{"url": "http://example.com"}]}, + ) + assert choice.finish_reason == "length" + assert choice.provider_specific_fields["native_finish_reason"] == "max_tokens" + assert choice.provider_specific_fields["citations"] == [{"url": "http://example.com"}] + + def test_gemini_safety_reason_exposed(self): + from litellm.types.utils import Choices + + choice = Choices(finish_reason="SAFETY") + assert choice.finish_reason == "content_filter" + assert choice.provider_specific_fields["native_finish_reason"] == "SAFETY" + + def test_anthropic_tool_use_reason_exposed(self): + from litellm.types.utils import Choices + + choice = Choices(finish_reason="tool_use") + assert choice.finish_reason == "tool_calls" + assert choice.provider_specific_fields["native_finish_reason"] == "tool_use" + + def test_max_tokens_reason_exposed(self): + from litellm.types.utils import Choices + + choice = Choices(finish_reason="MAX_TOKENS") + assert choice.finish_reason == "length" + assert choice.provider_specific_fields["native_finish_reason"] == "MAX_TOKENS"