diff --git a/litellm/__init__.py b/litellm/__init__.py index 0fe3d091c2a..b14acb0f1e3 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -328,6 +328,7 @@ anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = ( ) openai_system_messages_first: bool = False disable_vertex_batch_output_transformation: bool = False +disable_inline_reasoning_extraction: bool = False extra_spend_tag_headers: Optional[List[str]] = None in_memory_llm_clients_cache: "LLMClientCache" safe_memory_mode: bool = False diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 60be4d44706..b0110c5880c 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2230,7 +2230,7 @@ def extract_reasoning_content(message: Mapping[str, object]) -> tuple[str | None return _provider_text_or_none(message["reasoning_content"]), _provider_text_or_none(message_content) elif "reasoning" in message: return _provider_text_or_none(message["reasoning"]), _provider_text_or_none(message_content) - elif isinstance(message_content, str): + elif isinstance(message_content, str) and not litellm.disable_inline_reasoning_extraction: return parse_content_for_reasoning(message_content) return None, _provider_text_or_none(message_content) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index cd3727401ef..ca199d2790a 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -2495,7 +2495,11 @@ class AmazonConverseConfig(BaseConfig): ( extracted_reasoning_content_str, _content_str, - ) = parse_content_for_reasoning(content["text"]) + ) = ( + (None, content["text"]) + if litellm.disable_inline_reasoning_extraction + else parse_content_for_reasoning(content["text"]) + ) if _content_str is not None: content_str += _content_str if "toolUse" in content: diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 0c601a6b2c8..9cd3c278658 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -2837,6 +2837,41 @@ class TestConvertToModelResponseObjectCompletion: assert result.choices[0].message.content == "The answer is 4." assert result.choices[0].message.reasoning_content == "2+2=4" + @staticmethod + def _inline_thinking_response() -> dict[str, object]: + return { + "id": "chatcmpl-5", + "model": "gpt-5.6", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "2+2=4The answer is 4.", "role": "assistant"}, + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 10, "total_tokens": 15}, + } + + def test_inline_thinking_tag_moves_to_reasoning_content_by_default(self): + result = convert_to_model_response_object( + response_object=self._inline_thinking_response(), + model_response_object=ModelResponse(), + ) + assert (result.choices[0].message.content, result.choices[0].message.reasoning_content) == ( + "The answer is 4.", + "2+2=4", + ) + + def test_inline_thinking_tag_stays_in_content_when_inline_extraction_disabled(self, monkeypatch): + monkeypatch.setattr(litellm, "disable_inline_reasoning_extraction", True) + result = convert_to_model_response_object( + response_object=self._inline_thinking_response(), + model_response_object=ModelResponse(), + ) + message = result.choices[0].message + assert message.content == "2+2=4The answer is 4." + assert getattr(message, "reasoning_content", None) is None, message + def test_reasoning_content_not_mirrored_into_provider_specific_fields(self): """Mirroring reasoning_content into provider_specific_fields made cache-replayed messages diverge from live Anthropic messages, which diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index cc0ef062e4b..200709dce80 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -9,6 +9,7 @@ from typing import Final, cast import pytest +import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( ENCRYPTED_REASONING_SIGNATURE_PREFIX, TOOL_RESULT_IMAGE_BOUNDARY, @@ -18,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( converse_redacted_content, encrypted_content_from_signature, encrypted_reasoning_signature, + extract_reasoning_content, get_file_ids_from_messages, get_format_from_file_id, get_semantic_cache_prompt_from_messages, @@ -2463,3 +2465,31 @@ def test_schema_has_unresolvable_ref_flags_refs_that_cannot_be_inlined(schema, u def test_schema_size_counts_every_written_object_once(schema, properties, branches): assert schema_property_count(schema) == properties assert schema_grammar_branch_count(schema) == branches + + +_INLINE_REASONING_TEXTS: Final = ( + "step 1Final answer: 42", + "step 1Final answer: 42", + "step 1Final answer: 42", +) + + +@pytest.mark.parametrize("text", _INLINE_REASONING_TEXTS) +def test_extract_reasoning_content_moves_leading_tag_span_by_default(text: str) -> None: + assert extract_reasoning_content({"role": "assistant", "content": text}) == ("step 1", "Final answer: 42") + + +@pytest.mark.parametrize("text", _INLINE_REASONING_TEXTS) +def test_extract_reasoning_content_keeps_leading_tag_span_when_inline_extraction_disabled( + text: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_inline_reasoning_extraction", True) + assert extract_reasoning_content({"role": "assistant", "content": text}) == (None, text) + + +def test_extract_reasoning_content_still_returns_native_reasoning_when_inline_extraction_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_inline_reasoning_extraction", True) + message: Final = {"role": "assistant", "content": "inlineanswer", "reasoning_content": "native"} + assert extract_reasoning_content(message) == ("native", "inlineanswer") diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 1fb5542c964..846bae5e5d4 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -10739,3 +10739,89 @@ def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_ assert route.call_count == expected_calls assert all("outputConfig" in json.loads(call.request.content) for call in route.calls[:1]) assert all("outputConfig" not in json.loads(call.request.content) for call in route.calls[1:]) + + +_INLINE_THINKING_TEXT: Final = "step 1, step 2Final answer: 42" + + +def _converse_response(content_blocks: list[ContentBlock]) -> ModelResponse: + raw_response: Final = httpx.Response( + 200, + json={ + "output": {"message": {"role": "assistant", "content": content_blocks}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 10, "outputTokens": 20, "totalTokens": 30}, + "metrics": {"latencyMs": 1}, + }, + request=httpx.Request( + "POST", "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-opus-5-5/converse" + ), + ) + return AmazonConverseConfig()._transform_response( + model="us.anthropic.claude-opus-5-5", + response=raw_response, + model_response=ModelResponse(), + stream=False, + logging_obj=None, + optional_params={}, + api_key=None, + data=None, + messages=[], + encoding=None, + ) + + +def test_converse_moves_leading_thinking_tag_to_reasoning_content_by_default() -> None: + message: Final = _converse_response([{"text": _INLINE_THINKING_TEXT}]).choices[0].message + + assert (message.content, message.reasoning_content) == ("Final answer: 42", "step 1, step 2") + + +def test_converse_keeps_leading_thinking_tag_in_content_when_inline_extraction_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_inline_reasoning_extraction", True) + response: Final = _converse_response([{"text": _INLINE_THINKING_TEXT}]) + + assert response.choices[0].message.model_dump() == { + "content": _INLINE_THINKING_TEXT, + "role": "assistant", + "tool_calls": None, + "function_call": None, + "provider_specific_fields": None, + } + details: Final = response.usage.completion_tokens_details + assert (details.reasoning_tokens, details.text_tokens) == (0, 20), "all 20 output tokens are answer text" + + +def test_converse_still_surfaces_native_reasoning_block_when_inline_extraction_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_inline_reasoning_extraction", True) + message: Final = ( + _converse_response( + [ + {"reasoningContent": {"reasoningText": {"text": "native reasoning", "signature": "sig"}}}, + {"text": _INLINE_THINKING_TEXT}, + ] + ) + .choices[0] + .message + ) + + assert (message.content, message.reasoning_content) == (_INLINE_THINKING_TEXT, "native reasoning") + assert message.thinking_blocks == [{"type": "thinking", "thinking": "native reasoning", "signature": "sig"}] + + +def test_converse_non_streaming_content_matches_streaming_when_inline_extraction_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder + + monkeypatch.setattr(litellm, "disable_inline_reasoning_extraction", True) + streamed: Final = AWSEventStreamDecoder(model="us.anthropic.claude-opus-5-5").converse_chunk_parser( + {"contentBlockIndex": 0, "delta": {"text": _INLINE_THINKING_TEXT}} + ) + non_streamed: Final = _converse_response([{"text": _INLINE_THINKING_TEXT}]) + + assert non_streamed.choices[0].message.content == streamed.choices[0].delta.content == _INLINE_THINKING_TEXT diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index 1a0bbe171a9..7645508a304 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -168,6 +168,21 @@ class TestOllamaChatConfigResponseFormat: assert result["messages"][0]["content"] == "Hello world!" assert result["messages"][0]["role"] == "user" + def test_transform_request_keeps_history_think_tags_out_of_thinking_when_inline_extraction_disabled( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "disable_inline_reasoning_extraction", True) + messages: Final = cast( + list[AllMessageValues], + [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "planHello"}], + ) + + result: Final = OllamaChatConfig().transform_request( + model="qwen3", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert result["messages"][1] == {"role": "assistant", "content": "planHello", "images": []} + def test_transform_request_empty_content_list(self): """Test handling of empty content list in transform_request""" config = OllamaChatConfig()