From 5a392b9e5f22b230be19f561d4e12e3f099781bc Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:07:11 -0800 Subject: [PATCH] fix(bedrock)!: keep Claude's thinking when response_format json_schema is set (#49696) * fix(anthropic): stop forcing the response_format json tool when Claude will think A forced tool call skips Claude's thinking. The json_schema emulation forced json_tool_call unless thinking.type was enabled, so adaptive thinking and the models that think by default (Opus 5, Sonnet 5, Haiku 5.5, and the always-on families) lost their thinking on every structured-output request. AnthropicModelInfo.response_format_tool_can_be_forced is now the one check the Anthropic, Bedrock Converse (including the native output fallback) and Databricks paths use. It leaves the tool unforced when thinking is requested in any form or the model thinks by default, and keeps forcing it when the request turns thinking off (thinking disabled or reasoning_effort none) on a model that accepts forced tools. The new thinking_on_by_default cost map key marks Opus 5, Sonnet 5 and Haiku 5.5 on the providers that reach these paths, and a miss-only fallback rule covers unmapped Claude 5 and later ids. * fix(bedrock): route native output and thinking token counts by whether Claude will think Bedrock's native output opt-in now uses the same will-think check as the json tool, so Claude 5 models that think by default with no thinking param go native. Converse also asks for /usage/output_tokens_details on those models, so reasoning tokens are reported for them. * fix(anthropic): tell Claude to answer through the unforced response_format tool Moves the integration tests that check the forced JSON tool from Claude 5 models to Opus 4.8, since Claude 5 models now keep their default thinking and take the unforced tool or native output * fix(anthropic): keep open-object schemas on the json tool when Claude will think Native output closes every object, so an open object came back empty. Also declare thinking_on_by_default in the Rust model catalog and expect the Converse thinking-token field in test_main * fix(anthropic): hold streamed prose until the JSON tool decides the answer In json mode with the unforced JSON tool, Claude can write its working before it calls the tool. The stream now holds that text, drops it when the JSON tool answers, and sends it when Claude calls a caller tool or skips the JSON tool, matching the non-streamed reply. * test(anthropic): build the stream test events without nested comprehensions * fix(anthropic): keep always-on thinking models unforced, open objects native when thinking, and the tool instruction Claude-only * test(anthropic): keep main's open-object integration cases and cover the unforced tool on a default-thinking model * fix(anthropic): treat thinking False as thinking off so the JSON tool stays forced Cover thinking False on the Anthropic and Converse forcing tests, and pin that a Converse json_schema stream with the unforced JSON tool never sends Claude's working text as content * fix(anthropic): drop the thinking False forcing case that completion never reaches litellm.completion turns thinking False into no thinking param before any route sees it, so the case was dead on the request path and False keeps meaning the model's default * test(bedrock): collect the Converse json_schema stream chunks in one assignment --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 3 + litellm/llms/anthropic/chat/handler.py | 40 +++- litellm/llms/anthropic/chat/transformation.py | 31 +++- litellm/llms/anthropic/common_utils.py | 28 +++ .../bedrock/chat/converse_transformation.py | 59 ++++-- .../anthropic_claude_transformation.py | 17 +- .../bedrock/chat/mantle/transformation.py | 6 +- litellm/llms/bedrock/common_utils.py | 8 +- .../llms/databricks/chat/transformation.py | 13 +- ...odel_prices_and_context_window_backup.json | 58 ++++++ litellm/types/utils.py | 1 + litellm/utils.py | 1 + model_prices_and_context_window.json | 58 ++++++ ...t_bedrock_native_structured_output_wire.py | 12 +- ...vertex_anthropic_structured_output_wire.py | 58 ++++-- .../test_reasoning_effort_levels_wire.py | 15 +- ...anthropic_json_tool_cut_stream_sync_sdk.py | 2 +- ...t_completions_response_format_anthropic.py | 22 +-- ...st_chat_completions_streaming_anthropic.py | 22 +-- .../test_fallback_generalizations.py | 17 ++ .../chat/test_anthropic_chat_handler.py | 116 +++++++++++- .../test_anthropic_chat_transformation.py | 85 +++++++++ ...mations_anthropic_claude_transformation.py | 35 ++-- .../chat/test_converse_transformation.py | 173 ++++++++++++++---- .../llms/bedrock/chat/test_invoke_handler.py | 102 +++++++++++ .../test_databricks_chat_transformation.py | 45 +++++ tests/unit/test_main.py | 1 + tests/unit/test_utils.py | 1 + 28 files changed, 887 insertions(+), 142 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 32b47bc6b46..50e090eabaf 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1673,6 +1673,9 @@ OPENAI_FINISH_REASONS: Final = [ ] HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = get_env_int("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call +RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION: Final = ( + "When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text." +) ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07" diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 4eae06dbcdf..c273c715ba6 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -734,6 +734,7 @@ class ModelResponseIterator: self._split_answer_blocks: tuple[str, ...] = () self._withheld_answer_close: str = "" self._streamed_answer_keys: frozenset[str] | None = None + self._held_prose: str = "" # For handling partial JSON chunks from fragmentation # See: https://github.com/BerriAI/litellm/issues/17473 @@ -826,8 +827,13 @@ class ModelResponseIterator: thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] = [] if "text" in content_block["delta"]: - text = content_block["delta"]["text"] - return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content + return ( + self._hold_prose(content_block["delta"]["text"]), + tool_use, + thinking_blocks, + provider_specific_fields, + reasoning_content, + ) self.content_blocks.append(content_block) if "partial_json" in content_block["delta"]: # Only emit tool calls if we're in a tool_use or server_tool_use block @@ -1025,7 +1031,7 @@ class ModelResponseIterator: if self.current_content_block_type in _THINKING_BLOCK_TYPES: self.thinking_block_index += 1 if content_block_start["content_block"]["type"] == "text": - text = self._release_answer_close() + content_block_start["content_block"]["text"] + text = self._release_answer_close() + self._hold_prose(content_block_start["content_block"]["text"]) elif ( content_block_start["content_block"]["type"] == "tool_use" or content_block_start["content_block"]["type"] == "server_tool_use" @@ -1161,7 +1167,7 @@ class ModelResponseIterator: elif type_chunk == "message_delta": finish_reason, usage, container, stop_reason_fields = self._handle_message_delta(chunk) provider_specific_fields.update(stop_reason_fields) - text = self._release_answer_close() + self._drain_split_answer_blocks() + text = self._release_answer_close() + self._drain_split_answer_blocks() + self._release_held_prose() if container: provider_specific_fields["container"] = container # Store container_id and re-emit code_interpreter_results @@ -1211,7 +1217,9 @@ class ModelResponseIterator: status_code=500, # it looks like Anthropic API does not return a status code in the chunk error - default to 500 ) - text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use) + text, tool_use = self._handle_json_mode_chunk( + text=text + self._settle_held_prose(tool_use), tool_use=tool_use + ) refusal_fields: Final = refusal_params(provider_specific_fields) returned_chunk: Final = ModelResponseStream( @@ -1309,6 +1317,24 @@ class ModelResponseIterator: self._response_format_verdict = _envelope_verdict(buffered, schema, RESPONSE_FORMAT_TOOL_NAME) return self._withhold_answer_close(buffered) if self._response_format_verdict == "pass_through" else "" + def _hold_prose(self, text: str) -> str: + if not self.json_mode or self.response_format_schema is None: + return text + self._held_prose += text + return "" + + def _settle_held_prose(self, tool_use: ChatCompletionToolCallChunk | None) -> str: + if tool_use is None or tool_use.get("id") is None or self.current_content_block_type != "tool_use": + return "" + held: Final = self._held_prose + self._held_prose = "" + return "" if tool_use.get("function", {}).get("name") == RESPONSE_FORMAT_TOOL_NAME else held + + def _release_held_prose(self) -> str: + held: Final = self._held_prose + self._held_prose = "" + return "" if self.converted_response_format_tool else held + def _withhold_answer_close(self, text: str) -> str: pending: Final = self._withheld_answer_close + text body: Final = pending.rstrip() @@ -1362,7 +1388,9 @@ class ModelResponseIterator: def cut_stream_chunk(self) -> ModelResponseStream | None: unsent: Final = self._flush_response_format_buffer() if self._response_format_buffer.getvalue() else "" - text: Final = unsent + self._release_answer_close() + self._drain_split_answer_blocks() + text: Final = ( + unsent + self._release_answer_close() + self._drain_split_answer_blocks() + self._release_held_prose() + ) if not text: return None return ModelResponseStream( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 09d16181c3a..4f890ed922f 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -26,6 +26,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.litellm_core_utils.asyncify import is_event_loop_running from litellm.litellm_core_utils.core_helpers import ( @@ -1786,7 +1787,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): json_schema: Mapping[str, object] | None, request_params: Mapping[str, object], ) -> bool: - open_object_takes_forced_json_tool: Final = ( + open_object_takes_json_tool: Final = ( json_schema is not None and schema_has_open_object(json_schema) and not self.is_thinking_requested(request_params) @@ -1794,7 +1795,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): and not request_params.get("tools") and not AnthropicModelInfo.forced_tool_use_unsupported(model) ) - return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool + return self.uses_native_structured_output(model) and not open_object_takes_json_tool def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None: return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema))) @@ -1809,7 +1810,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema)) def map_response_format_to_anthropic_tool( - self, value: dict | None, optional_params: dict, is_thinking_enabled: bool + self, value: dict | None, optional_params: dict, tool_is_forced: bool ) -> AnthropicMessagesTool | None: ignore_response_format_types: Final = ["text"] if value is None or value["type"] in ignore_response_format_types: # value is a no-op @@ -1827,6 +1828,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _tool: Final = self._create_json_tool_call_for_response_format( json_schema=json_schema, + description=None if tool_is_forced else RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) return _tool @@ -1929,7 +1931,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): drop_params=drop_params, runs_hosted_mcp=self.runs_hosted_mcp, ) - is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=request_params) + response_format_tool_is_forced: Final = ( + "response_format" in request_params + and AnthropicModelInfo.response_format_tool_can_be_forced( + model, self._resolved_provider, REQUEST_PARAMS.validate_python(request_params) + ) + ) # NB: ``map_openai_params`` deliberately does NOT sanitize tool names # here. Names are the *original* OpenAI names at this stage, and must @@ -1989,10 +1996,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if _output_format is not None: optional_params["output_format"] = _output_format else: - _tool = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled) + _tool = self.map_response_format_to_anthropic_tool( + value, optional_params, response_format_tool_is_forced + ) if _tool is None: continue - if not is_thinking_enabled and not AnthropicModelInfo.forced_tool_use_unsupported(model): + if response_format_tool_is_forced: _tool_choice = { "name": RESPONSE_FORMAT_TOOL_NAME, "type": "tool", @@ -2121,12 +2130,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def _create_json_tool_call_for_response_format( self, json_schema: dict | None = None, + description: str | None = None, ) -> AnthropicMessagesTool: """ Handles creating a tool call for getting responses in JSON format. Args: json_schema (Optional[dict]): The JSON schema the response should be in + description (Optional[str]): The tool description, left off the tool when None Returns: AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format @@ -2143,11 +2154,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): else: _input_schema.update(cast(AnthropicInputSchema, json_schema)) - _tool: Final = AnthropicMessagesTool( - name=RESPONSE_FORMAT_TOOL_NAME, - input_schema=_input_schema, + if description is None: + return AnthropicMessagesTool(name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema) + return AnthropicMessagesTool( + name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema, description=description ) - return _tool def should_strip_billing_metadata(self) -> bool: """ diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index bfd1341711a..bc9bd769d83 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1141,6 +1141,34 @@ class AnthropicModelInfo(BaseLLMModelInfo): """ return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider) + @staticmethod + def thinks_by_default(model: str, custom_llm_provider: str) -> bool: + return AnthropicModelInfo._is_always_on_thinking_model( + model, custom_llm_provider + ) or AnthropicModelInfo._supports_model_capability(model, "thinking_on_by_default", custom_llm_provider) + + @staticmethod + def will_think(model: str, custom_llm_provider: str, request_params: Mapping[str, object]) -> bool: + if AnthropicModelInfo._is_always_on_thinking_model(model, custom_llm_provider): + return True + match request_params: + case {"thinking": True | {"type": "enabled" | "adaptive"}}: + return True + case {"reasoning_effort": effort} if effort is not None and effort != "none": + return True + case {"thinking": {"type": "disabled"}} | {"reasoning_effort": "none"}: + return False + case _: + return AnthropicModelInfo.thinks_by_default(model, custom_llm_provider) + + @staticmethod + def response_format_tool_can_be_forced( + model: str, custom_llm_provider: str, request_params: Mapping[str, object] + ) -> bool: + return not AnthropicModelInfo.forced_tool_use_unsupported(model) and not AnthropicModelInfo.will_think( + model, custom_llm_provider, request_params + ) + @staticmethod def _supports_legacy_thinking(model: str, custom_llm_provider: str) -> bool: """Whether ``model`` is an adaptive-thinking model that still accepts legacy diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 056f34f1bf4..03fbc35a97f 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -22,6 +22,7 @@ from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_hea from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.integrations.anthropic_cache_control_hook import ( MAX_CACHE_CONTROL_BLOCKS, @@ -950,19 +951,24 @@ class AmazonConverseConfig(BaseConfig): def _supports_native_structured_outputs( model: str, custom_llm_provider: str | None, + will_think: bool, thinking_requested: bool, model_info: Mapping[str, object] | None, ) -> bool: """Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request. ``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and - ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when the - request asks for thinking, which the json tool path would drop. Without thinking, a model that cannot - force the json tool gets it unless that key is false. + ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when Claude + will think, which the json tool path would drop. When the request does not ask for thinking, a model + that cannot force the json tool gets it unless that key is false. Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html """ return bedrock_chat_uses_native_structured_output( - model, thinking_requested, custom_llm_provider or "bedrock_converse", model_info + model, + will_think=will_think, + thinking_requested=thinking_requested, + custom_llm_provider=custom_llm_provider or "bedrock_converse", + model_info=model_info, ) @staticmethod @@ -1005,11 +1011,13 @@ class AmazonConverseConfig(BaseConfig): self, model: str, json_schema: Mapping[str, object], + *, + will_think: bool, thinking_requested: bool, model_info: Mapping[str, object] | None, ) -> Mapping[str, object] | None: if not self._supports_native_structured_outputs( - model, self.custom_llm_provider, thinking_requested, model_info + model, self.custom_llm_provider, will_think, thinking_requested, model_info ): return None if BedrockModelInfo.get_base_model(model).startswith("anthropic"): @@ -1095,7 +1103,6 @@ class AmazonConverseConfig(BaseConfig): request_params: Final = tool_params_without_hosted_mcp( non_default_params, model=model, route="bedrock", drop_params=drop_params ) - is_thinking_enabled: Final = self.is_thinking_enabled(request_params) base_model: Final = BedrockModelInfo.get_base_model(model) drop_thinking_param: Final = self._is_deepseek_model(model=model, base_model=base_model) drop_reasoning_effort_param: Final = self._model_rejects_reasoning_effort_param( @@ -1109,7 +1116,6 @@ class AmazonConverseConfig(BaseConfig): model=model, optional_params=optional_params, non_default_params=request_params, - is_thinking_enabled=is_thinking_enabled, model_info=model_info, ) if param == "max_tokens" or param == "max_completion_tokens": @@ -1314,7 +1320,6 @@ class AmazonConverseConfig(BaseConfig): model: str, optional_params: dict, non_default_params: dict, - is_thinking_enabled: bool, model_info: Mapping[str, object] | None = None, ) -> dict: """ @@ -1339,9 +1344,16 @@ class AmazonConverseConfig(BaseConfig): if "type" in value and value["type"] == "text": return optional_params - thinking_requested: Final = self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)) + request_params: Final = REQUEST_PARAMS.validate_python(non_default_params) + thinking_requested: Final = self.is_thinking_requested(request_params) native_schema: Final = ( - self._native_output_schema(model, json_schema, thinking_requested, model_info) + self._native_output_schema( + model, + json_schema, + will_think=AnthropicModelInfo.will_think(model, "bedrock", request_params), + thinking_requested=thinking_requested, + model_info=model_info, + ) if json_schema is not None else None ) @@ -1359,17 +1371,24 @@ class AmazonConverseConfig(BaseConfig): elif json_schema is not None: # Fallback: translate to a synthetic tool call # https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode + tool_is_forced: Final = litellm.utils.supports_tool_choice( + model=model, custom_llm_provider=self.custom_llm_provider + ) and AnthropicModelInfo.response_format_tool_can_be_forced(model, "bedrock", request_params) + schema_description: Final = description if isinstance(description, str) else None + instructs_unforced_tool: Final = not tool_is_forced and BedrockModelInfo.get_base_model(model).startswith( + "anthropic" + ) _tool: Final = self._create_json_tool_call_for_response_format( json_schema=json_schema, - description=description, + description=( + " ".join(filter(None, (RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, schema_description))) + if instructs_unforced_tool + else schema_description + ), ) optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) - if ( - litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider) - and not is_thinking_enabled - and not AnthropicModelInfo.forced_tool_use_unsupported(model) - ): + if tool_is_forced: optional_params["tool_choice"] = ToolChoiceValuesBlock( tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME) ) @@ -1380,8 +1399,8 @@ class AmazonConverseConfig(BaseConfig): # returns {} instead of the requested JSON. The model already # returns JSON when the prompt asks for it. - # A request that goes native only because the model can't force the json tool keeps the json tool's faked - # stream, so it still needs only bedrock:InvokeModel. + # A request that goes native without asking for thinking keeps the json tool's faked stream, so it still + # needs only bedrock:InvokeModel, unless the model takes native output on every request. streams_natively: Final = native_schema is not None and bedrock_chat_opts_into_native_structured_output( model, thinking_requested, self.custom_llm_provider or "bedrock_converse", model_info ) @@ -2219,8 +2238,8 @@ class AmazonConverseConfig(BaseConfig): } if additional_request_params: data["additionalModelRequestFields"] = additional_request_params - if "thinking" in additional_request_params: - data["additionalModelResponseFieldPaths"] = ("/usage/output_tokens_details",) + if "thinking" in additional_request_params or AnthropicModelInfo.thinks_by_default(model, "bedrock"): + data["additionalModelResponseFieldPaths"] = ("/usage/output_tokens_details",) if system_content_blocks: data["system"] = system_content_blocks diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index dc85028440f..62e1bee5217 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -142,14 +142,19 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): json_schema: Mapping[str, object] | None, request_params: Mapping[str, object], ) -> bool: - return self.chat_uses_native_structured_output( - model, self.is_thinking_requested(request_params), self._deployment_model_info - ) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None) + return self.chat_uses_native_structured_output(model, request_params, self._deployment_model_info) and ( + json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None + ) def chat_uses_native_structured_output( - self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None + self, model: str, request_params: Mapping[str, object], model_info: Mapping[str, object] | None ) -> bool: - return bedrock_chat_uses_native_structured_output(model, thinking_requested, model_info=model_info) + return bedrock_chat_uses_native_structured_output( + model, + will_think=self.will_think(model, "bedrock", request_params), + thinking_requested=self.is_thinking_requested(request_params), + model_info=model_info, + ) def uses_opt_in_native_output( self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] @@ -359,7 +364,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): request_body=anthropic_request, model_info=model_info, chat_uses_native_output=lambda: self.chat_uses_native_structured_output( - model, self.is_thinking_requested(_NON_DEFAULT_PARAMS.validate_python(filtered_params)), model_info + model, _NON_DEFAULT_PARAMS.validate_python(filtered_params), model_info ), ) strip_unsupported_bedrock_invoke_output_config_keys( diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index 952ca7a2cd4..7e635a61d4d 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -55,9 +55,11 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): ) def chat_uses_native_structured_output( - self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None + self, model: str, request_params: Mapping[str, object], model_info: Mapping[str, object] | None ) -> bool: - return bedrock_chat_opts_into_native_structured_output(model, thinking_requested, model_info=model_info) + return bedrock_chat_opts_into_native_structured_output( + model, self.will_think(model, "bedrock", request_params), model_info=model_info + ) def _default_max_tokens_models(self, model: str, litellm_params: Mapping[str, object]) -> tuple[str, ...]: return (f"bedrock_mantle/{model}", *super()._default_max_tokens_models(model, litellm_params)) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index c1b3b5616b8..dfee28a7cc9 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -356,13 +356,15 @@ def bedrock_chat_opts_into_native_structured_output( def bedrock_chat_uses_native_structured_output( model: str, + *, + will_think: bool, thinking_requested: bool, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None, ) -> bool: - return bedrock_chat_opts_into_native_structured_output( - model, thinking_requested, custom_llm_provider, model_info - ) or (not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info)) + return bedrock_chat_opts_into_native_structured_output(model, will_think, custom_llm_provider, model_info) or ( + not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info) + ) def bedrock_native_output_is_deployment_opt_in( diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index d943718ed73..859211bd40d 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -31,6 +31,7 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.base_utils import ( type_to_response_format_param, # pyright: ignore[reportUnknownVariableType] # base_utils helper returns an untyped dict ) +from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.databricks import ( AllDatabricksContentValues, @@ -541,12 +542,12 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): model: str, value: dict | None, optional_params: dict, - is_thinking_enabled: bool, + tool_is_forced: bool, ) -> DatabricksTool | None: if value is None: return None - tool: Final = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled) + tool: Final = self.map_response_format_to_anthropic_tool(value, optional_params, tool_is_forced) databricks_tool: Final = self.convert_anthropic_tool_to_databricks_tool(tool) return databricks_tool @@ -579,7 +580,6 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): if "claude" in model else non_default_params ) - is_thinking_enabled: Final = self.is_thinking_enabled(request_params) mapped_params: Final = super().map_openai_params(request_params, optional_params, model, drop_params) if "claude" in model: AnthropicConfig.translate_legacy_thinking_for_adaptive_model( @@ -596,17 +596,20 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): mapped_params.pop("max_completion_tokens", None) if "response_format" in request_params and "claude" in model: + tool_is_forced: Final = AnthropicConfig.response_format_tool_can_be_forced( + model, "databricks", REQUEST_PARAMS.validate_python(request_params) + ) _tool: Final = self.map_response_format_to_databricks_tool( model, request_params["response_format"], mapped_params, - is_thinking_enabled, + tool_is_forced, ) if _tool is not None: self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) optional_params["json_mode"] = True - if not is_thinking_enabled: + if tool_is_forced: _tool_choice: Final = ChatCompletionToolChoiceObjectParam( type="function", function=ChatCompletionToolChoiceFunctionParam(name=RESPONSE_FORMAT_TOOL_NAME), diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4d576443b30..a292ce0d14e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2125,6 +2125,7 @@ "anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -2213,6 +2214,7 @@ "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -2301,6 +2303,7 @@ "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2389,6 +2392,7 @@ "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2472,6 +2476,7 @@ "au.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2560,6 +2565,7 @@ "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2958,6 +2964,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3002,6 +3009,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3046,6 +3054,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3090,6 +3099,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3134,6 +3144,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3178,6 +3189,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -4063,6 +4075,7 @@ "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, "litellm_provider": "azure_ai", @@ -4204,6 +4217,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -16507,6 +16521,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -17042,6 +17057,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -21910,6 +21926,7 @@ "prompt_cache_min_tokens": 512, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -22068,6 +22085,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -50492,6 +50510,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -50569,6 +50588,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -52974,6 +52994,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -53012,6 +53033,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -53261,6 +53283,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -60636,6 +60659,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -64770,6 +64794,14 @@ "thinking_always_on": true } }, + { + "name": "claude-thinking-on-by-default", + "pattern": "claude-[a-z]+-[5-9](?!\\d)", + "description": "Claude at major 5 or higher, in any id shape that contains claude--. Anthropic's thinking docs list every Claude 5+ model as thinking by default (on or always on) when the request sends no thinking param, and every 4.x as off. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. The response_format json tool is then left unforced, since a forced tool call skips the model's thinking.", + "model_info": { + "thinking_on_by_default": true + } + }, { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", @@ -69293,6 +69325,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69368,6 +69401,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69624,6 +69658,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69689,6 +69724,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -75440,6 +75476,7 @@ "apac.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "litellm_provider": "bedrock_converse", "supports_tool_search": true, @@ -75556,6 +75593,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77060,6 +77098,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77112,6 +77151,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77156,6 +77196,7 @@ "thinking_always_on": false, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77203,6 +77244,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77239,6 +77281,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77295,6 +77338,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77341,6 +77385,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77381,6 +77426,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77422,6 +77468,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77468,6 +77515,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77519,6 +77567,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77565,6 +77614,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77625,6 +77675,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77676,6 +77727,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77726,6 +77778,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77770,6 +77823,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -79529,6 +79583,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -79743,6 +79798,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -79909,6 +79965,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -81894,6 +81951,7 @@ "prompt_cache_min_tokens": 512.0, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_anthropic_thinking_payload": true, "supports_assistant_prefill": false, "supports_function_calling": true, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4c47489e08f..ba29a67a36c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -224,6 +224,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_adaptive_thinking: bool | None supports_legacy_thinking: ReadOnly[bool | None] thinking_always_on: ReadOnly[bool | None] + thinking_on_by_default: ReadOnly[bool | None] supports_tool_search: bool | None supports_mid_conversation_system: bool | None supports_url_context: bool | None diff --git a/litellm/utils.py b/litellm/utils.py index 4add9d53cb3..7f81ece8314 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7581,6 +7581,7 @@ def get_model_info_helper( supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None), supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None), thinking_always_on=_model_info.get("thinking_always_on", None), + thinking_on_by_default=_model_info.get("thinking_on_by_default", None), supports_tool_search=_model_info.get("supports_tool_search", None), supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4d576443b30..a292ce0d14e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2125,6 +2125,7 @@ "anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -2213,6 +2214,7 @@ "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -2301,6 +2303,7 @@ "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2389,6 +2392,7 @@ "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2472,6 +2476,7 @@ "au.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2560,6 +2565,7 @@ "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2958,6 +2964,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3002,6 +3009,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3046,6 +3054,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3090,6 +3099,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3134,6 +3144,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3178,6 +3189,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -4063,6 +4075,7 @@ "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, "litellm_provider": "azure_ai", @@ -4204,6 +4217,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -16507,6 +16521,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -17042,6 +17057,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -21910,6 +21926,7 @@ "prompt_cache_min_tokens": 512, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -22068,6 +22085,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -50492,6 +50510,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -50569,6 +50588,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -52974,6 +52994,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -53012,6 +53033,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -53261,6 +53283,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -60636,6 +60659,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -64770,6 +64794,14 @@ "thinking_always_on": true } }, + { + "name": "claude-thinking-on-by-default", + "pattern": "claude-[a-z]+-[5-9](?!\\d)", + "description": "Claude at major 5 or higher, in any id shape that contains claude--. Anthropic's thinking docs list every Claude 5+ model as thinking by default (on or always on) when the request sends no thinking param, and every 4.x as off. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. The response_format json tool is then left unforced, since a forced tool call skips the model's thinking.", + "model_info": { + "thinking_on_by_default": true + } + }, { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", @@ -69293,6 +69325,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69368,6 +69401,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69624,6 +69658,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -69689,6 +69724,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -75440,6 +75476,7 @@ "apac.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "litellm_provider": "bedrock_converse", "supports_tool_search": true, @@ -75556,6 +75593,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77060,6 +77098,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77112,6 +77151,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77156,6 +77196,7 @@ "thinking_always_on": false, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77203,6 +77244,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77239,6 +77281,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77295,6 +77338,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77341,6 +77385,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77381,6 +77426,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77422,6 +77468,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77468,6 +77515,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77519,6 +77567,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77565,6 +77614,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77625,6 +77675,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77676,6 +77727,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -77726,6 +77778,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -77770,6 +77823,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -79529,6 +79583,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -79743,6 +79798,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -79909,6 +79965,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -81894,6 +81951,7 @@ "prompt_cache_min_tokens": 512.0, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_anthropic_thinking_payload": true, "supports_assistant_prefill": false, "supports_function_calling": true, diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index 67edcf7f3e4..2502bd90bd6 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -622,10 +622,15 @@ def test_messages_thinking_on_a_bedrock_converse_native_model_gets_native_output @pytest.mark.parametrize("route", ["converse", "invoke"]) -def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gateway: Gateway, route: str) -> None: +@pytest.mark.parametrize( + ("backend", "thinks_by_default"), [(_OPUS_4_8, False), (_HAIKU, True)], ids=["opus_4_8", "haiku_5_5"] +) +def test_bedrock_native_model_with_no_thinking_param_gets_native_output_only_when_it_thinks_by_default( + gateway: Gateway, route: str, backend: str, thinks_by_default: bool +) -> None: marker, marker_hex = _marker() with wire_server(_bedrock) as wire, gateway.scenario() as scenario: - model: Final = _deployment(scenario, wire, route, _HAIKU) + model: Final = _deployment(scenario, wire, route, backend) response: Final = gateway.request( "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA)) ) @@ -634,6 +639,9 @@ def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gatewa assert not message.get("tool_calls"), response.text _, body = _one_call(wire, marker_hex) invoke: Final = route == "invoke" + if thinks_by_default: + _assert_native_without_tools(body, invoke, _SCHEMA) + return assert _native_schema(body, invoke) is None, body assert _tools(body, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index db81ff70263..19928ecd6e0 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -34,14 +34,20 @@ from pydantic import JsonValue, TypeAdapter pytestmark: Final = pytest.mark.shares_worker_state _JSON_TOOL: Final = "json_tool_call" +_UNFORCED_JSON_TOOL_DESCRIPTION: Final = ( + "When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text." +) _USER_TOOL: Final = "lookup" _STRUCTURED_OUTPUTS_BETA: Final = "structured-outputs-2025-11-13" _TOOL_MODEL: Final = "claude-sonnet-5" _EFFORT_GATED_MODEL: Final = "claude-haiku-4-5" _NATIVE_MODEL: Final = "claude-opus-5-5" -_FORCED_TOOL_NATIVE_MODEL: Final = "claude-haiku-5-5" +_FORCED_TOOL_NATIVE_MODEL: Final = "claude-opus-4-8" +_THINKING_BY_DEFAULT_NATIVE_MODEL: Final = "claude-haiku-5-5" +_THINKING_OFF_TOOL_MODEL: Final = "claude-opus-4-8" _UNMAPPED_MODEL: Final = "claude-scripted-9" _BEDROCK_INVOKE_MODEL: Final = "anthropic.claude-sonnet-5" +_BEDROCK_TOOL_PATH_MODEL: Final = "anthropic.claude-opus-4-8" _BEDROCK_JSON_TOOL_MODEL: Final = "us.anthropic.claude-opus-4-8" _THINKING_SIGNATURE: Final = "c2NyaXB0ZWQtc2lnbmF0dXJl" _PROJECT: Final = "scripted-project" @@ -608,11 +614,15 @@ def _one_delivery(wire: Wire, marker_hex: str) -> _Delivery: return sent -def _json_tool_schema(delivery: _Delivery) -> dict[str, JsonValue]: +def _json_tool(delivery: _Delivery) -> dict[str, JsonValue]: tools: Final = tuple(object_value(tool) for tool in list_value(delivery.body["tools"])) (json_tool,) = tuple(tool for tool in tools if tool["name"] == _JSON_TOOL) - assert json_tool.keys() == {"name", "input_schema"}, json_tool - return object_value(json_tool["input_schema"]) + assert json_tool.keys() - {"description"} == {"name", "input_schema"}, json_tool + return json_tool + + +def _json_tool_schema(delivery: _Delivery) -> dict[str, JsonValue]: + return object_value(_json_tool(delivery)["input_schema"]) def _folded_format(delivery: _Delivery) -> dict[str, JsonValue]: @@ -830,10 +840,10 @@ def _family_deployment(gateway: Gateway, scenario: Scenario, api_base: str, memb case "azure_ai_unmapped": return scenario.model(model=f"azure_ai/{_UNMAPPED_MODEL}", api_base=api_base, api_key=_AZURE_KEY) case "bedrock_invoke": - return _bedrock_invoke_deployment(scenario, api_base, _BEDROCK_INVOKE_MODEL) + return _bedrock_invoke_deployment(scenario, api_base, _BEDROCK_TOOL_PATH_MODEL) case "bedrock_mantle": return scenario.model( - model=f"bedrock_mantle/{_BEDROCK_INVOKE_MODEL}", + model=f"bedrock_mantle/{_BEDROCK_TOOL_PATH_MODEL}", api_base=api_base, api_key=_MANTLE_KEY, aws_region_name="us-east-1", @@ -1448,21 +1458,24 @@ def test_tool_path_stream_cut_before_the_tool_block_closes_still_delivers_the_an _one_delivery(wire, marker_hex) -@pytest.mark.parametrize("accepts_forced_tools", [True, False], ids=["accepts_forced_tools", "rejects_forced_tools"]) -def test_vertex_chat_json_schema_forces_the_json_tool_only_when_the_model_accepts_forced_tools( - gateway: Gateway, accepts_forced_tools: bool +@pytest.mark.parametrize( + ("tool_model", "forced"), + [(_THINKING_OFF_TOOL_MODEL, True), (_TOOL_MODEL, False), (None, False)], + ids=["accepts_forced_tools", "thinks_by_default", "rejects_forced_tools"], +) +def test_vertex_chat_json_schema_forces_the_json_tool_only_when_the_model_will_not_think( + gateway: Gateway, tool_model: str | None, forced: bool ) -> None: marker, marker_hex = _marker("plain") - backend: Final = ( - _TOOL_MODEL if accepts_forced_tools else _claude_rejecting_forced_tools(gateway, "vertex_ai/", "claude") - ) + backend: Final = tool_model or _claude_rejecting_forced_tools(gateway, "vertex_ai/", "claude") with wire_server(_peer) as wire, gateway.scenario() as scenario: model: Final = _vertex_deployment(gateway, scenario, wire.url, backend) response: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, marker)) assert _chat_content(response) == _payload(marker_hex), response.text sent: Final = _one_delivery(wire, marker_hex) assert _json_tool_schema(sent)["properties"] == _SCHEMA["properties"], sent.body - assert sent.body.get("tool_choice") == (_FORCED_JSON_TOOL if accepts_forced_tools else None), sent.body + assert sent.body.get("tool_choice") == (_FORCED_JSON_TOOL if forced else None), sent.body + assert _json_tool(sent).get("description") == (None if forced else _UNFORCED_JSON_TOOL_DESCRIPTION), sent.body assert "output_config" not in sent.body and "output_format" not in sent.body, sent.body @@ -1566,6 +1579,25 @@ def test_native_model_json_schema_with_an_open_object_takes_the_json_tool_and_ke sent: Final = _one_delivery(wire, marker_hex) assert not _structured(sent.body), sent.body assert _json_tool_schema(sent)["properties"] == _VALUES_SCHEMA["properties"], sent.body + assert sent.body.get("tool_choice") == _FORCED_JSON_TOOL, sent.body + + +@pytest.mark.parametrize("provider", ["anthropic", "azure_ai"]) +def test_open_object_schema_takes_the_unforced_json_tool_on_a_model_that_thinks_by_default( + gateway: Gateway, provider: str +) -> None: + marker, marker_hex = _marker("values") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _native_deployment(gateway, scenario, wire.url, provider, _THINKING_BY_DEFAULT_NATIVE_MODEL) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_VALUES_RESPONSE_FORMAT) + ) + assert _chat_content(response) == _tool_input("values", marker_hex), response.text + sent: Final = _one_delivery(wire, marker_hex) + assert not _structured(sent.body), sent.body + assert _json_tool_schema(sent)["properties"] == _VALUES_SCHEMA["properties"], sent.body + assert _json_tool(sent).get("description") == _UNFORCED_JSON_TOOL_DESCRIPTION, sent.body + assert "tool_choice" not in sent.body, sent.body @pytest.mark.parametrize( diff --git a/tests/integration/routing/test_reasoning_effort_levels_wire.py b/tests/integration/routing/test_reasoning_effort_levels_wire.py index d15867ac572..efca26d38d0 100644 --- a/tests/integration/routing/test_reasoning_effort_levels_wire.py +++ b/tests/integration/routing/test_reasoning_effort_levels_wire.py @@ -1977,6 +1977,9 @@ _INVOKE_NATIVE_OUTPUT: Final[dict[str, JsonValue]] = { "format": {"type": "json_schema", "schema": _NATIVE_OUTPUT_SCHEMA}, } _JSON_TOOL: Final = "json_tool_call" +_UNFORCED_JSON_TOOL_DESCRIPTION: Final = ( + "When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text." +) _CONVERSE_TOOL_SCHEMA: Final[dict[str, JsonValue]] = { key: value for key, value in _NATIVE_OUTPUT_SCHEMA.items() if key != "additionalProperties" } @@ -1994,7 +1997,7 @@ _CONVERSE_WITH_JSON_TOOL: Final = _with_fields( { "toolSpec": { "name": _JSON_TOOL, - "description": _JSON_TOOL, + "description": _UNFORCED_JSON_TOOL_DESCRIPTION, "inputSchema": {"json": _CONVERSE_TOOL_SCHEMA}, } } @@ -2004,7 +2007,15 @@ _CONVERSE_WITH_JSON_TOOL: Final = _with_fields( ) _INVOKE_WITH_JSON_TOOL: Final = _with_fields( _without(_invoke_chat("high", _ADAPTIVE, ()), "anthropic_beta"), - {"tools": [{"name": _JSON_TOOL, "input_schema": _NATIVE_OUTPUT_SCHEMA}]}, + { + "tools": [ + { + "name": _JSON_TOOL, + "description": _UNFORCED_JSON_TOOL_DESCRIPTION, + "input_schema": _NATIVE_OUTPUT_SCHEMA, + } + ] + }, ) diff --git a/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py b/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py index 54a6df02edf..294791128ab 100644 --- a/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py +++ b/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py @@ -27,7 +27,7 @@ _RESPONSE_FORMAT: Final[dict[str, JsonValue]] = { "json_schema": {"name": "risk", "schema": _SCHEMA}, } _ANTHROPIC_MODEL: Final = "anthropic/claude-scripted-9" -_BEDROCK_INVOKE_MODEL: Final = "bedrock/invoke/anthropic.claude-sonnet-5" +_BEDROCK_INVOKE_MODEL: Final = "bedrock/invoke/anthropic.claude-opus-4-8" def _payload(marker_hex: str) -> dict[str, JsonValue]: diff --git a/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py b/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py index 07e4358ff97..5cb7a7eee8e 100644 --- a/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py +++ b/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py @@ -6,7 +6,7 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.chat_completions.bases.anthropic import CLAUDE_HAIKU_5_5_TEST_CASE +from integration.translation.chat_completions.bases.anthropic import CLAUDE_OPUS_4_8_TEST_CASE from integration.translation.runner import assert_translation PLACE_SCHEMA: Final = { @@ -24,25 +24,25 @@ PLACE_TOOL_USE: Final = { "caller": {"type": "direct"}, } -CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace( - CLAUDE_HAIKU_5_5_TEST_CASE, +CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE: Final = replace( + CLAUDE_OPUS_4_8_TEST_CASE, scenario="json_schema", litellm_request={ - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "max_tokens": 128, "messages": [{"role": "user", "content": PLACE_PROMPT}], "response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": PLACE_SCHEMA}}, "cache": {"no-cache": True}, }, expected_provider_request={ - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "messages": [{"role": "user", "content": [{"type": "text", "text": PLACE_PROMPT}]}], "max_tokens": 128, "tool_choice": {"name": "json_tool_call", "type": "tool"}, "tools": [{"name": "json_tool_call", "input_schema": PLACE_SCHEMA}], }, mock_provider_response={ - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "id": "msg_011CgCoP2UR3vcM5ZWmEBd8D", "type": "message", "role": "assistant", @@ -68,7 +68,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace( "id": ANY, "created": ANY, "object": "chat.completion", - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "choices": [ { "finish_reason": "stop", @@ -97,11 +97,11 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace( }, ) -CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace( - CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE, +CLAUDE_OPUS_4_8_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace( + CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE, scenario="json_schema_tool_named_envelope", mock_provider_response={ - **CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE.mock_provider_response, + **CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE.mock_provider_response, "content": [ { **PLACE_TOOL_USE, @@ -114,7 +114,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace( @pytest.mark.parametrize( "case", - [CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE, CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE], + [CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE, CLAUDE_OPUS_4_8_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE], ids=lambda case: case.id, ) def test_chat_completions_response_format_anthropic( diff --git a/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py b/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py index fca8de69f00..a41752a96cf 100644 --- a/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py +++ b/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py @@ -6,7 +6,7 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import SSEFrame, StreamingTranslationTestCase from integration.translation.chat_completions.bases.anthropic import ( - CLAUDE_HAIKU_5_5_TEST_CASE, + CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, ) from integration.translation.runner import assert_streaming_translation @@ -158,27 +158,27 @@ def _place_chunk(delta: dict[str, JsonValue]) -> SSEFrame: "id": ANY, "object": "chat.completion.chunk", "created": ANY, - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "choices": [{"index": 0, "delta": delta}], } ) -CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationTestCase( +CLAUDE_OPUS_4_8_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationTestCase( scenario="json_schema_cut_stream", - litellm_endpoint=CLAUDE_HAIKU_5_5_TEST_CASE.litellm_endpoint, + litellm_endpoint=CLAUDE_OPUS_4_8_TEST_CASE.litellm_endpoint, litellm_request={ - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "max_tokens": 128, "messages": [{"role": "user", "content": PLACE_PROMPT}], "response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": PLACE_SCHEMA}}, "cache": {"no-cache": True}, "stream": True, }, - expected_provider_endpoint=CLAUDE_HAIKU_5_5_TEST_CASE.expected_provider_endpoint, - expected_provider_headers=CLAUDE_HAIKU_5_5_TEST_CASE.expected_provider_headers, + expected_provider_endpoint=CLAUDE_OPUS_4_8_TEST_CASE.expected_provider_endpoint, + expected_provider_headers=CLAUDE_OPUS_4_8_TEST_CASE.expected_provider_headers, expected_provider_request={ - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "messages": [{"role": "user", "content": [{"type": "text", "text": PLACE_PROMPT}]}], "max_tokens": 128, "tool_choice": {"name": "json_tool_call", "type": "tool"}, @@ -191,7 +191,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT data={ "type": "message_start", "message": { - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "id": "msg_011CgCoP5RkexwE6N7PRZEca", "type": "message", "role": "assistant", @@ -263,7 +263,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT "id": ANY, "object": "chat.completion", "created": ANY, - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "choices": [ { "index": 0, @@ -276,7 +276,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT @pytest.mark.parametrize( "case", - [CLAUDE_SONNET_5_5_STREAMING_TEST_CASE, CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE], + [CLAUDE_SONNET_5_5_STREAMING_TEST_CASE, CLAUDE_OPUS_4_8_JSON_SCHEMA_CUT_STREAM_TEST_CASE], ids=lambda case: case.id, ) def test_chat_completions_streaming_anthropic( diff --git a/tests/unit/litellm_core_utils/test_fallback_generalizations.py b/tests/unit/litellm_core_utils/test_fallback_generalizations.py index 327031cfaf9..d8e56c0ae51 100644 --- a/tests/unit/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/unit/litellm_core_utils/test_fallback_generalizations.py @@ -563,6 +563,23 @@ def test_shipped_rules_flag_unmapped_fable_as_always_on_thinking(shipped_cost_ma assert other.get("thinking_always_on") is None +@pytest.mark.parametrize( + ("model", "provider", "on_by_default"), + [ + ("claude-opus-5-9", "anthropic", True), + ("claude-newfamily-6", "anthropic", True), + ("us.anthropic.claude-sonnet-5-9", "bedrock", True), + ("claude-opus-4-9", "anthropic", None), + ], +) +def test_shipped_rules_flag_unmapped_claude_5_and_later_as_thinking_by_default( + shipped_cost_map, model, provider, on_by_default +): + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider=provider) + assert info.get("thinking_on_by_default") is on_by_default + + @pytest.mark.parametrize( "model,provider", [ diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 7e2786327dc..e81ee2a4e02 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -3140,6 +3140,118 @@ def test_streamed_json_mode_truncated_output_is_passed_through_untouched(): assert "".join(contents) == '{"name": "json_tool_call", "parameters": {"tier": 3' +def _block_events( + index: int, block: dict[str, object], deltas: tuple[dict[str, object], ...] +) -> tuple[dict[str, object], ...]: + return ( + {"type": "content_block_start", "index": index, "content_block": block}, + *({"type": "content_block_delta", "index": index, "delta": delta} for delta in deltas), + {"type": "content_block_stop", "index": index}, + ) + + +def _message_events(*blocks: tuple[dict[str, object], tuple[dict[str, object], ...]]) -> tuple[dict[str, object], ...]: + starts_and_deltas: Final = tuple( + chain.from_iterable(_block_events(index, block, deltas) for index, (block, deltas) in enumerate(blocks)) + ) + return ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + }, + *starts_and_deltas, + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 40}, + }, + ) + + +_THINKING_BLOCK: Final = ( + {"type": "thinking", "thinking": "", "signature": ""}, + ({"type": "thinking_delta", "thinking": "7 * 142 = 994"}, {"type": "signature_delta", "signature": "sig"}), +) +_PREAMBLE_BLOCK: Final = ( + {"type": "text", "text": ""}, + ({"type": "text_delta", "text": "**Working it out:** "}, {"type": "text_delta", "text": "the count is **120**"}), +) + + +def _json_tool_block(answer: dict[str, object]) -> tuple[dict[str, object], tuple[dict[str, object], ...]]: + encoded: Final = json.dumps(answer) + return ( + {"type": "tool_use", "id": "toolu_1", "name": RESPONSE_FORMAT_TOOL_NAME, "input": {}}, + tuple({"type": "input_json_delta", "partial_json": part} for part in (encoded[:5], encoded[5:])), + ) + + +def _parse_stream( + events: tuple[dict[str, object], ...], schema: dict[str, object] | None = _STREAMED_PERSON_SCHEMA +) -> tuple[ModelResponseStream, ...]: + iterator: Final = ModelResponseIterator( + streaming_response=MagicMock(), + sync_stream=True, + json_mode=True, + response_format_schema=None if schema is None else response_format_schema(schema), + ) + return tuple(iterator.chunk_parser(event) for event in events) + + +def _joined_content(chunks: tuple[ModelResponseStream, ...]) -> str: + return "".join(chunk.choices[0].delta.content or "" for chunk in chunks) + + +def test_streamed_json_mode_drops_prose_written_before_the_json_tool(): + chunks: Final = _parse_stream(_message_events(_THINKING_BLOCK, _PREAMBLE_BLOCK, _json_tool_block(_STREAMED_PERSON))) + + assert json.loads(_joined_content(chunks)) == _STREAMED_PERSON + assert "".join(getattr(chunk.choices[0].delta, "reasoning_content", None) or "" for chunk in chunks) == ( + "7 * 142 = 994" + ) + assert all(chunk.choices[0].delta.tool_calls is None for chunk in chunks) + + +def test_streamed_json_mode_returns_prose_when_claude_skips_the_json_tool(): + chunks: Final = _parse_stream(_message_events(_THINKING_BLOCK, _PREAMBLE_BLOCK)) + + assert _joined_content(chunks) == "**Working it out:** the count is **120**" + + +def test_streamed_json_mode_sends_prose_ahead_of_a_caller_tool_call(): + caller_tool: Final = ( + {"type": "tool_use", "id": "toolu_2", "name": "get_weather", "input": {}}, + ({"type": "input_json_delta", "partial_json": '{"city": "Paris"}'},), + ) + + chunks: Final = _parse_stream(_message_events(_PREAMBLE_BLOCK, caller_tool)) + tool_chunk_index: Final = next(i for i, chunk in enumerate(chunks) if chunk.choices[0].delta.tool_calls) + + assert _joined_content(chunks[: tool_chunk_index + 1]) == "**Working it out:** the count is **120**" + assert chunks[tool_chunk_index].choices[0].delta.tool_calls[0].function.name == "get_weather" + + +def test_streamed_native_output_text_is_not_held_back(): + answer: Final = json.dumps(_STREAMED_PERSON) + text_block: Final = ( + {"type": "text", "text": ""}, + tuple({"type": "text_delta", "text": part} for part in (answer[:6], answer[6:])), + ) + + chunks: Final = _parse_stream(_message_events(text_block), schema=None) + + assert tuple(chunk.choices[0].delta.content for chunk in chunks[2:4]) == (answer[:6], answer[6:]) + + def test_streamed_json_mode_tool_name_envelope_is_unwrapped_when_name_is_a_schema_property(): envelope = json.dumps({"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _STREAMED_CONTACT}) @@ -3565,7 +3677,7 @@ def test_streamed_json_mode_partial_objects_that_repeat_a_field_keep_the_order_t assert "".join(contents) == "".join(blocks) -def test_streamed_json_mode_text_after_the_answer_follows_the_whole_answer(): +def test_streamed_json_mode_text_after_the_answer_is_dropped_like_the_non_streamed_reply(): answer = json.dumps(_STREAMED_RISK_OBJECT) message_start, *_, message_delta = _json_tool_events(()) text_block = ( @@ -3576,7 +3688,7 @@ def test_streamed_json_mode_text_after_the_answer_follows_the_whole_answer(): chunks = _json_mode_stream_chunks((message_start, *_json_tool_block_events(0, answer), *text_block, message_delta)) - assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == answer + " Done." + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == answer def test_streamed_json_mode_envelope_whose_label_is_a_property_name_is_still_unwrapped(): diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index f1c38ef4e64..80845b1f12d 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -22,6 +22,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt @@ -1424,6 +1425,33 @@ def test_dict_fields_keep_native_output_when_the_json_tool_would_not_be_forced(m assert "json_tool_call" not in [tool["name"] for tool in optional_params.get("tools", [])] +def test_dict_fields_take_the_unforced_json_tool_on_a_default_thinking_model(local_model_cost_map): + optional_params: Final = AnthropicConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "out", + "schema": { + "type": "object", + "properties": {"field": {"type": "object", "additionalProperties": {"type": "integer"}}}, + "required": ["field"], + }, + }, + } + }, + optional_params={}, + model="claude-opus-5", + drop_params=False, + ) + + assert "output_format" not in optional_params + assert "tool_choice" not in optional_params + assert [ + (tool["name"], tool["input_schema"]["properties"]["field"]) for tool in optional_params["tools"] + ] == [("json_tool_call", {"type": "object", "additionalProperties": {"type": "integer"}})] + + def test_native_structured_output_uses_bundled_capability_when_remote_map_lags( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -6956,6 +6984,63 @@ def test_response_format_tool_path_skips_forced_tool_choice_when_unsupported(loc assert "tool_choice" not in result +@pytest.mark.parametrize( + ("model", "thinking_params", "forced"), + [ + ("claude-opus-4-8", {"thinking": {"type": "adaptive"}}, False), + ("claude-opus-5", {}, False), + ("claude-sonnet-5", {}, False), + ("claude-haiku-5-5", {}, False), + ("claude-fable-5", {"reasoning_effort": "none"}, False), + ("claude-opus-4-8", {}, True), + ("claude-opus-5", {"thinking": {"type": "disabled"}}, True), + ("claude-haiku-5-5", {"reasoning_effort": "none"}, True), + ], + ids=[ + "opus-4-8-adaptive", + "opus-5-default-thinking", + "sonnet-5-default-thinking", + "haiku-5-5-default-thinking", + "fable-5-always-on-reasoning-effort-none", + "opus-4-8-default-off", + "opus-5-thinking-disabled", + "haiku-5-5-reasoning-effort-none", + ], +) +def test_response_format_tool_is_forced_only_when_the_model_will_not_think( + set_native_structured_output, model, thinking_params, forced +): + set_native_structured_output(model, None) + config: Final = AnthropicConfig() + optional_params: Final = config.map_openai_params( + non_default_params={ + **thinking_params, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "capital", + "schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, + }, + }, + optional_params={}, + model=model, + drop_params=False, + ) + + request: Final = config.transform_request( + model=model, + messages=[{"role": "user", "content": "Name the capital of France."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert [tool["name"] for tool in request["tools"]] == ["json_tool_call"] + assert (request.get("tool_choice") == {"name": "json_tool_call", "type": "tool"}) is forced + assert request["tools"][0].get("description") == (None if forced else RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION) + + def _eager_chat_function(**extra: object) -> dict[str, object]: return { "name": "write_file", diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index 34a4cf5225b..93f6e237fc7 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -562,19 +562,27 @@ def _invoke_params_for_schema( @pytest.mark.parametrize( - ("native_key", "thinking_params", "native"), + ("model", "native_key", "thinking_params", "native"), [ - ("legacy", {}, True), - ("bedrock", {}, False), - ("bedrock", {"thinking": {"type": "adaptive"}}, True), - ("bedrock", {"reasoning_effort": "low"}, True), + ("us.anthropic.claude-haiku-5-5", "legacy", {}, True), + ("us.anthropic.claude-haiku-5-5", "bedrock", {}, True), + ("us.anthropic.claude-opus-4-8", "bedrock", {}, False), + ("us.anthropic.claude-opus-4-8", "bedrock", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-opus-4-8", "bedrock", {"reasoning_effort": "low"}, True), + ("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "disabled"}}, False), + ], + ids=[ + "legacy-key", + "bedrock-key-thinking-by-default", + "bedrock-key-thinking-off-by-default", + "bedrock-key-adaptive", + "bedrock-key-effort", + "bedrock-key-thinking-disabled", ], - ids=["legacy-key", "bedrock-key-no-thinking", "bedrock-key-adaptive", "bedrock-key-effort"], ) -def test_bedrock_native_output_key_applies_only_when_thinking_is_requested( - set_native_structured_output, native_key, thinking_params, native +def test_bedrock_native_output_key_applies_only_when_the_model_will_think( + set_native_structured_output, model, native_key, thinking_params, native ): - model = "us.anthropic.claude-haiku-5-5" set_native_structured_output(model, native_key) schema = {"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]} @@ -914,9 +922,7 @@ def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_th assert result["output_config"] == {"effort": "high"} -def test_bedrock_chat_invoke_adaptive_thinking_keeps_the_forced_response_format_tool(set_native_structured_output): - """Anthropic accepts a forced tool_choice next to adaptive thinking, and dropping - it lets the model answer in prose instead of the requested JSON.""" +def test_bedrock_chat_invoke_adaptive_thinking_leaves_the_response_format_tool_unforced(set_native_structured_output): set_native_structured_output("us.anthropic.claude-opus-5", None) result = AmazonAnthropicClaudeConfig().map_openai_params( non_default_params={ @@ -935,7 +941,8 @@ def test_bedrock_chat_invoke_adaptive_thinking_keeps_the_forced_response_format_ drop_params=False, ) - assert result["tool_choice"] == {"type": "tool", "name": result["tools"][0]["name"]} + assert [tool["name"] for tool in result["tools"]] == ["json_tool_call"] + assert "tool_choice" not in result assert result["thinking"] == {"type": "adaptive"} @@ -2084,7 +2091,7 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co (False, True, None, {"thinking": {"type": "adaptive"}}, True), (False, False, None, {}, False), (False, None, {"supports_bedrock_native_structured_output": False}, {}, False), - (True, True, None, {}, False), + (True, True, None, {"thinking": {"type": "disabled"}}, False), ], ids=[ "cannot-force-tool", diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 7c19ef5b751..b8eb5858fda 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -15,6 +15,7 @@ import respx import litellm from litellm._logging import verbose_logger from litellm import ModelResponse, Router, completion +from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, _bedrock_tools_pt, @@ -1440,18 +1441,27 @@ def test_thinking_request_adds_output_tokens_details_response_path(): assert result["additionalModelResponseFieldPaths"] == ("/usage/output_tokens_details",) -def test_request_without_thinking_omits_response_field_paths(): - config = AmazonConverseConfig() - - result = config._transform_request( - model="bedrock/converse/us.anthropic.claude-opus-4-7", +@pytest.mark.parametrize( + ("model", "thinks_by_default"), + [ + ("bedrock/converse/us.anthropic.claude-opus-4-7", False), + ("bedrock/converse/us.anthropic.claude-opus-5", True), + ("us.anthropic.claude-haiku-5-5", True), + ], + ids=["opus-4-7-thinking-off-by-default", "opus-5-thinking-on-by-default", "haiku-5-5-thinking-on-by-default"], +) +def test_request_without_thinking_asks_for_thinking_tokens_only_when_the_model_thinks_by_default( + local_model_cost_map: None, model: str, thinks_by_default: bool +): + result = AmazonConverseConfig()._transform_request( + model=model, messages=[{"role": "user", "content": "hi"}], optional_params={"maxTokens": 256}, litellm_params={}, headers={}, ) - assert "additionalModelResponseFieldPaths" not in result + assert (result.get("additionalModelResponseFieldPaths") == ("/usage/output_tokens_details",)) is thinks_by_default def test_transform_usage_prefers_provider_reasoning_tokens(): @@ -4937,7 +4947,6 @@ def test_translate_response_format_native_output_config(monkeypatch): model="anthropic.claude-sonnet-4-5-20250929-v1:0", optional_params=optional_params, non_default_params={"response_format": response_format}, - is_thinking_enabled=False, ) # Should have outputConfig, NOT tools @@ -4965,20 +4974,20 @@ def test_translate_response_format_native_output_config(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) -CAPITAL_RESPONSE_FORMAT: Final = { - "type": "json_schema", - "json_schema": { - "name": "capital", - "schema": { - "type": "object", - "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, - "required": ["city", "country"], - "additionalProperties": False, - }, +CAPITAL_JSON_SCHEMA: Final = { + "name": "capital", + "schema": { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"], + "additionalProperties": False, }, } +CAPITAL_RESPONSE_FORMAT: Final = {"type": "json_schema", "json_schema": CAPITAL_JSON_SCHEMA} + + ADAPTIVE_THINKING: Final = {"type": "adaptive"} @@ -5030,24 +5039,124 @@ def test_json_schema_shape_follows_each_inference_profile_own_native_output_flag assert "fake_stream" not in native_params assert "outputConfig" not in tool_request assert [tool["toolSpec"]["name"] for tool in tool_request["toolConfig"]["tools"]] == ["json_tool_call"] - assert tool_request["toolConfig"]["toolChoice"] == {"tool": {"name": "json_tool_call"}} + assert "toolChoice" not in tool_request["toolConfig"] assert tool_params.get("fake_stream", False) is stream +_JSON_TOOL_CHOICE: Final = {"tool": {"name": "json_tool_call"}} + + @pytest.mark.parametrize( - ("native_key", "thinking_params", "native"), + ("model", "native_key", "model_info", "thinking_params", "forced"), [ - ("legacy", {}, True), - ("bedrock", {}, False), - ("bedrock", {"thinking": ADAPTIVE_THINKING}, True), - ("bedrock", {"thinking": {"type": "enabled", "budget_tokens": 1024}}, True), - ("bedrock", {"reasoning_effort": "low"}, True), - ("bedrock", {"reasoning_effort": "none"}, False), - ("bedrock", {"thinking": {"type": "disabled"}}, False), + ( + "us.anthropic.claude-opus-4-8", + "bedrock", + {"supports_bedrock_native_structured_output": False}, + {"thinking": ADAPTIVE_THINKING}, + False, + ), + ("us.anthropic.claude-opus-5", None, None, {}, False), + ("global.anthropic.claude-sonnet-5", None, None, {}, False), + ("us.anthropic.claude-haiku-5-5", None, None, {}, False), + ("us.anthropic.claude-fable-5", None, None, {}, False), + ("us.anthropic.claude-fable-5", None, None, {"reasoning_effort": "none"}, False), + ("us.anthropic.claude-opus-4-8", None, None, {}, True), + ("us.anthropic.claude-opus-5", None, None, {"thinking": {"type": "disabled"}}, True), + ("us.anthropic.claude-haiku-5-5", None, None, {"reasoning_effort": "none"}, True), + ], + ids=[ + "opus-4-8-adaptive-native-output-fallback", + "opus-5-default-thinking", + "sonnet-5-default-thinking", + "haiku-5-5-default-thinking", + "fable-5-always-on", + "fable-5-always-on-reasoning-effort-none", + "opus-4-8-default-off", + "opus-5-thinking-disabled", + "haiku-5-5-reasoning-effort-none", + ], +) +def test_json_tool_is_forced_only_when_the_model_will_not_think( + set_native_structured_output, model, native_key, model_info, thinking_params, forced +): + set_native_structured_output(model, native_key) + config: Final = AmazonConverseConfig() + + response_format: Final = { + "type": "json_schema", + "json_schema": {**CAPITAL_JSON_SCHEMA, "description": "The capital of a country"}, + } + + optional_params: Final = config.map_openai_params( + non_default_params={"response_format": response_format, **thinking_params}, + optional_params={}, + model=model, + drop_params=False, + model_info=model_info, + ) + request: Final = config._transform_request( + model=model, + messages=[{"role": "user", "content": "Name the capital of France."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "outputConfig" not in request + assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] + assert (request["toolConfig"].get("toolChoice") == _JSON_TOOL_CHOICE) is forced + assert request["toolConfig"]["tools"][0]["toolSpec"]["description"] == ( + "The capital of a country" + if forced + else f"{RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION} The capital of a country" + ) + + +def test_unforced_json_tool_keeps_the_schema_description_on_a_non_claude_model(set_native_structured_output): + model: Final = "us.meta.llama3-3-70b-instruct-v1:0" + set_native_structured_output(model, None) + config: Final = AmazonConverseConfig() + + optional_params: Final = config.map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": {**CAPITAL_JSON_SCHEMA, "description": "The capital of a country"}, + } + }, + optional_params={}, + model=model, + drop_params=False, + ) + request: Final = config._transform_request( + model=model, + messages=[{"role": "user", "content": "Name the capital of France."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "toolChoice" not in request["toolConfig"] + assert [tool["toolSpec"]["description"] for tool in request["toolConfig"]["tools"]] == ["The capital of a country"] + + +@pytest.mark.parametrize( + ("model", "native_key", "thinking_params", "native"), + [ + ("us.anthropic.claude-haiku-5-5", "legacy", {}, True), + ("us.anthropic.claude-haiku-5-5", "bedrock", {}, True), + ("us.anthropic.claude-opus-4-8", "bedrock", {}, False), + ("us.anthropic.claude-opus-4-8", "bedrock", {"thinking": ADAPTIVE_THINKING}, True), + ("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "enabled", "budget_tokens": 1024}}, True), + ("us.anthropic.claude-opus-4-8", "bedrock", {"reasoning_effort": "low"}, True), + ("us.anthropic.claude-haiku-5-5", "bedrock", {"reasoning_effort": "none"}, False), + ("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "disabled"}}, False), ], ids=[ "legacy-key", - "bedrock-key-no-thinking", + "bedrock-key-thinking-by-default", + "bedrock-key-thinking-off-by-default", "bedrock-key-adaptive", "bedrock-key-enabled", "bedrock-key-effort", @@ -5055,10 +5164,9 @@ def test_json_schema_shape_follows_each_inference_profile_own_native_output_flag "bedrock-key-thinking-disabled", ], ) -def test_bedrock_native_output_key_applies_only_when_thinking_is_requested( - set_native_structured_output, native_key, thinking_params, native +def test_bedrock_native_output_key_applies_only_when_the_model_will_think( + set_native_structured_output, model, native_key, thinking_params, native ): - model: Final = "us.anthropic.claude-haiku-5-5" set_native_structured_output(model, native_key) _, request = _converse_request_for_json_schema(model, False, thinking_params) @@ -5078,7 +5186,7 @@ _FORCED_JSON_TOOL_CHOICE: Final = {"tool": {"name": "json_tool_call"}} (False, True, None, {"thinking": {"type": "adaptive"}}, True, None), (False, False, None, None, False, None), (False, None, {"supports_bedrock_native_structured_output": False}, None, False, None), - (True, True, None, None, False, _FORCED_JSON_TOOL_CHOICE), + (True, True, None, {"thinking": {"type": "disabled"}}, False, _FORCED_JSON_TOOL_CHOICE), ], ids=[ "cannot-force-tool", @@ -5596,7 +5704,6 @@ def test_translate_response_format_fallback_tool_call(): model="anthropic.claude-3-haiku-20240307-v1:0", optional_params=optional_params, non_default_params={"response_format": response_format}, - is_thinking_enabled=False, ) # Should use tool-call approach, NOT outputConfig (model doesn't support native structured outputs) @@ -5633,7 +5740,6 @@ def test_native_structured_output_no_fake_stream(monkeypatch): model="anthropic.claude-sonnet-4-5-20250929-v1:0", optional_params=optional_params, non_default_params={"response_format": response_format, "stream": True}, - is_thinking_enabled=False, ) assert "outputConfig" in result @@ -5954,7 +6060,6 @@ def test_json_object_no_schema_skips_tool_injection(monkeypatch): model="anthropic.claude-sonnet-4-5-20250929-v1:0", optional_params=optional_params, non_default_params=non_default_params, - is_thinking_enabled=False, ) # Should NOT use native outputConfig (no schema provided) diff --git a/tests/unit/llms/bedrock/chat/test_invoke_handler.py b/tests/unit/llms/bedrock/chat/test_invoke_handler.py index 996cfec6e22..4424f7640a3 100644 --- a/tests/unit/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/unit/llms/bedrock/chat/test_invoke_handler.py @@ -1149,6 +1149,108 @@ async def test_converse_streaming_completion_yields_text_and_finish_reason( _assert_converse_stream_text_and_finish_reason(sync_chunks) +_CONVERSE_PROSE: Final = "Let me work this out. " + + +async def _stream_chunks( + request: Mapping[str, object], transport: httpx.MockTransport, async_mode: bool +) -> tuple[ModelResponseStream, ...]: + if not async_mode: + return tuple(litellm.completion(**request, client=HTTPHandler(client=httpx.Client(transport=transport)))) + async_stream: Final = await litellm.acompletion(**request, client=AsyncHTTPHandler(transport=transport)) + return tuple([chunk async for chunk in async_stream]) + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize( + ("tool_block", "stop_reason", "expected_content", "expected_tools", "expected_finish_reason"), + [ + pytest.param( + {"toolUse": {"toolUseId": "tooluse_1", "name": "json_tool_call", "input": {"answer": 120}}}, + "tool_use", + '{"answer": 120}', + (), + "stop", + id="prose_then_json_tool", + ), + pytest.param( + {"toolUse": {"toolUseId": "tooluse_1", "name": "get_weather", "input": {"city": "Paris"}}}, + "tool_use", + _CONVERSE_PROSE, + (("get_weather", '{"city": "Paris"}'),), + "tool_calls", + id="prose_then_caller_tool", + ), + pytest.param(None, "end_turn", _CONVERSE_PROSE, (), "stop", id="json_tool_skipped"), + ], +) +@pytest.mark.asyncio +async def test_converse_json_schema_stream_sends_claude_prose_only_when_claude_skips_the_json_tool( + _aws_test_credentials: None, + async_mode: bool, + tool_block: Mapping[str, object] | None, + stop_reason: str, + expected_content: str, + expected_tools: tuple[tuple[str, str], ...], + expected_finish_reason: str, +) -> None: + converse_reply: Final = { + "output": { + "message": { + "role": "assistant", + "content": [{"text": _CONVERSE_PROSE}, *((tool_block,) if tool_block is not None else ())], + } + }, + "stopReason": stop_reason, + "usage": {"inputTokens": 30, "outputTokens": 20, "totalTokens": 50}, + "metrics": {"latencyMs": 1}, + } + + def handle_request(request: httpx.Request) -> httpx.Response: + body: Final = json.loads(request.content) + assert "toolChoice" not in body["toolConfig"] + assert "json_tool_call" in [tool["toolSpec"]["name"] for tool in body["toolConfig"]["tools"]] + return httpx.Response(200, json=converse_reply) + + transport: Final = httpx.MockTransport(handle_request) + request: Final = { + "model": "bedrock/us.anthropic.claude-opus-5", + "messages": [{"role": "user", "content": "What is the LCM of 24 and 120? Reply in JSON."}], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": {"type": "object", "properties": {"answer": {"type": "integer"}}, "required": ["answer"]}, + }, + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, + } + ], + "stream": True, + } + chunks: Final = await _stream_chunks(request, transport, async_mode) + + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == expected_content + tool_calls: Final = tuple( + itertools.chain.from_iterable(chunk.choices[0].delta.tool_calls or () for chunk in chunks) + ) + named_calls: Final = tuple( + (call.function.name, "".join(c.function.arguments or "" for c in tool_calls)) + for call in tool_calls + if call.function.name + ) + assert named_calls == expected_tools + assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == [ + expected_finish_reason + ] + + def test_mistral_invoke_stream_yields_text_and_finish_reason( _aws_test_credentials: None, ) -> None: diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index d2b78235c29..e04dd35f851 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -19,6 +19,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.databricks.common_utils import DatabricksException @@ -2591,6 +2592,50 @@ def test_databricks_claude_json_schema_preserves_refs_in_tool_parameters() -> No ] +@pytest.mark.parametrize( + ("model", "thinking_params", "forced"), + ( + ("databricks-claude-opus-5", {}, False), + ("databricks-claude-opus-4-8", {"thinking": {"type": "adaptive"}}, False), + ("databricks-claude-opus-5-5", {}, False), + ("databricks-claude-opus-4-8", {}, True), + ("databricks-claude-opus-5", {"thinking": {"type": "disabled"}}, True), + ), + ids=( + "opus-5-default-thinking", + "opus-4-8-adaptive", + "opus-5-5-without-forced-tools", + "opus-4-8-default-off", + "opus-5-thinking-disabled", + ), +) +def test_databricks_claude_json_tool_is_forced_only_when_the_model_will_not_think( + local_model_cost_map: None, model: str, thinking_params: dict[str, object], forced: bool +) -> None: + optional_params: Final = litellm.utils.get_optional_params( + model=model, + custom_llm_provider="databricks", + response_format=copy.deepcopy(JSON_SCHEMA_RESPONSE_FORMAT), + **thinking_params, + ) + request: Final[dict[str, object]] = TypeAdapter(dict[str, object]).validate_python( + DatabricksConfig().transform_request( # pyright: ignore[reportUnknownMemberType] # inherited hook is untyped + model=model, + messages=[{"role": "user", "content": "Name the capital of France."}], + optional_params=dict(optional_params), + litellm_params={}, + headers={}, + ) + ) + + assert ( + request.get("tool_choice") == {"type": "function", "function": {"name": "json_tool_call"}} + ) is forced + assert optional_params["tools"][0]["function"].get("description") == ( + None if forced else RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION + ) + + def test_databricks_pydantic_json_schema_preserves_nested_refs() -> None: class Person(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 0220495032b..99c6f6c402c 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -7703,6 +7703,7 @@ async def test_bedrock_converse_parses_scripted_response( assert json.loads(route.calls[0].request.content) == { "messages": [{"role": "user", "content": [{"text": "hello"}]}], "inferenceConfig": {}, + "additionalModelResponseFieldPaths": ["/usage/output_tokens_details"], } assert response.choices[0].message.content == "Hello from Bedrock." assert response.usage.prompt_tokens == 8 diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index e293b2a5be3..e24499db071 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1244,6 +1244,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "supports_anthropic_thinking_payload": {"type": "boolean"}, "supports_legacy_thinking": {"type": "boolean"}, "thinking_always_on": {"type": "boolean"}, + "thinking_on_by_default": {"type": "boolean"}, "supports_mid_conversation_system": {"type": "boolean"}, "supports_sampling_params": {"type": "boolean"}, "supports_output_config": {"type": "boolean"},