From 47e6144288bd0ebedd041896ed37ee51c9f77b8b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:27:04 -0700 Subject: [PATCH] fix(responses): drop a malformed tool_choice sent with a stored prompt under drop_params A stored prompt can carry its own tools, so the check kept any tool_choice next to one. A value that is not 'auto', 'none', 'required' or an object is malformed whatever tools the prompt holds, so it now drops under drop_params. Also types the parameters of the tests this PR adds. --- .../llms/openai/responses/transformation.py | 11 +++- .../count_tokens/test_token_counter.py | 4 +- ...test_openai_count_tokens_transformation.py | 4 +- .../test_openai_responses_transformation.py | 58 +++++++++++++------ tests/unit/responses/test_responses_utils.py | 14 ++--- 5 files changed, 60 insertions(+), 31 deletions(-) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index e6dd20ef15b..340f158da46 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -319,6 +319,7 @@ def _bundled_openai_reasoning_models() -> frozenset[str]: _TOOL_CHOICES_WITHOUT_TOOLS: Final = frozenset({"auto", "none"}) +_TOOL_CHOICE_STRINGS: Final = _TOOL_CHOICES_WITHOUT_TOOLS | {"required"} _PROVIDERS_CHECKED_FOR_MALFORMED_VALUES: Final = (LlmProviders.OPENAI, LlmProviders.AZURE) @@ -355,9 +356,17 @@ def _names_a_sent_tool(tool_choice: object, tools: tuple[Mapping[str, object], . return True +def _has_tool_choice_shape(tool_choice: object) -> bool: + if isinstance(tool_choice, str): + return tool_choice in _TOOL_CHOICE_STRINGS + return as_str_mapping(tool_choice) is not None + + def _accepts_tool_choice(tool_choice: object, params: Mapping[str, object]) -> bool: - if tool_choice is None or params.get("prompt") is not None: + if tool_choice is None: return True + if params.get("prompt") is not None: + return _has_tool_choice_shape(tool_choice) tools: Final = params.get("tools") if isinstance(tool_choice, str): return tool_choice in _TOOL_CHOICES_WITHOUT_TOOLS or (tool_choice == "required" and bool(tools)) diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index cfb52bf9173..13c3c16e074 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -765,7 +765,7 @@ async def test_native_count_answers_openais_rejection_with_its_status_and_text( @pytest.mark.asyncio @pytest.mark.parametrize(("declared", "effort_counted"), [(True, True), (False, False)], ids=["declared", "undeclared"]) async def test_the_count_reads_the_deployments_declared_efforts_under_drop_params( - local_model_cost_map, monkeypatch: pytest.MonkeyPatch, declared: bool, effort_counted: bool + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, declared: bool, effort_counted: bool ) -> None: row: Final = litellm.model_cost["gpt-6.1-sol"] effort: Final = next( @@ -790,7 +790,7 @@ async def test_the_count_reads_the_deployments_declared_efforts_under_drop_param @pytest.mark.asyncio @pytest.mark.parametrize("drop_params", [True, False], ids=["drop-params", "send-as-is"]) async def test_the_count_rounds_the_effort_to_a_declared_level_as_inference_does( - local_model_cost_map, monkeypatch: pytest.MonkeyPatch, drop_params: bool + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, drop_params: bool ) -> None: declared: Final = ["low", "medium", "high"] monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py index 41e0c67348a..628ae7d1a83 100644 --- a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -683,7 +683,7 @@ def _disabled_effort(model: str) -> str: _DROP: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset()) -def test_the_count_drops_what_inference_drops_under_drop_params(local_model_cost_map) -> None: +def test_the_count_drops_what_inference_drops_under_drop_params(local_model_cost_map: None) -> None: malformed: Final = { "tool_choice": "required", "parallel_tool_calls": "true", @@ -700,7 +700,7 @@ def test_the_count_drops_what_inference_drops_under_drop_params(local_model_cost assert inference == {"reasoning": {"summary": "auto"}} -def test_the_count_keeps_an_effort_the_deployment_declares(local_model_cost_map) -> None: +def test_the_count_keeps_an_effort_the_deployment_declares(local_model_cost_map: None) -> None: effort: Final = _disabled_effort("gpt-6.1-sol") body: Final = native_count_request( diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 031f27a6699..7fac30486df 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -3352,11 +3352,21 @@ class TestMalformedParamsFollowDropParams: pytest.param({"tool_choice": 5, "tools": [_WEATHER_TOOL]}, id="tool-choice-a-number-with-tools"), pytest.param({"tool_choice": ["get_time"], "tools": [_WEATHER_TOOL]}, id="tool-choice-a-list-with-tools"), pytest.param({"reasoning": "high"}, id="reasoning-not-an-object"), + pytest.param( + {"tool_choice": 5, "prompt": {"id": "pmpt_abc"}}, id="tool-choice-a-number-with-stored-prompt" + ), + pytest.param( + {"tool_choice": "always", "prompt": {"id": "pmpt_abc"}}, id="unknown-string-with-stored-prompt" + ), ], ) def test_a_malformed_value_is_dropped_under_drop_params_and_sent_as_is_without( - self, local_model_cost_map, monkeypatch, config, params - ): + self, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + config: OpenAIResponsesAPIConfig, + params: dict[str, object], + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) malformed: Final = next(name for name in params if name != "tools") @@ -3367,8 +3377,8 @@ class TestMalformedParamsFollowDropParams: assert sent == params def test_azure_o_series_forwards_a_row_disabled_effort_while_dropping_malformed_values( - self, local_model_cost_map, monkeypatch - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: row: Final = litellm.model_cost["gpt-6.1-sol"] monkeypatch.setitem(litellm.model_cost, "gpt-6.1-sol", {**row, "supports_minimal_reasoning_effort": False}) @@ -3392,6 +3402,10 @@ class TestMalformedParamsFollowDropParams: pytest.param({"parallel_tool_calls": None}, id="parallel-tool-calls-null"), pytest.param({"previous_response_id": "resp_abc"}, id="previous-response-id-string"), pytest.param({"tool_choice": "required", "prompt": {"id": "pmpt_abc"}}, id="required-with-stored-prompt"), + pytest.param( + {"tool_choice": {"type": "function", "name": "lookup_order"}, "prompt": {"id": "pmpt_abc"}}, + id="function-from-stored-prompt", + ), pytest.param( {"tool_choice": {"type": "custom", "name": "other"}, "tools": [_SQL_TOOL]}, id="custom-with-a-custom-tool", @@ -3416,7 +3430,9 @@ class TestMalformedParamsFollowDropParams: ), ], ) - def test_a_well_formed_value_is_kept_under_drop_params(self, local_model_cost_map, params): + def test_a_well_formed_value_is_kept_under_drop_params( + self, local_model_cost_map: None, params: dict[str, object] + ) -> None: assert OpenAIResponsesAPIConfig().map_openai_params(dict(params), "gpt-6.1-sol", True) == params @pytest.mark.parametrize( @@ -3428,10 +3444,12 @@ class TestMalformedParamsFollowDropParams: ), ], ) - def test_a_tool_choice_the_check_cannot_read_is_kept_under_drop_params(self, local_model_cost_map, params): + def test_a_tool_choice_the_check_cannot_read_is_kept_under_drop_params( + self, local_model_cost_map: None, params: dict[str, object] + ) -> None: assert OpenAIResponsesAPIConfig().map_openai_params(dict(params), "gpt-6.1-sol", True) == params - def test_providers_outside_openai_and_azure_keep_these_values_as_sent(self, local_model_cost_map): + def test_providers_outside_openai_and_azure_keep_these_values_as_sent(self, local_model_cost_map: None) -> None: params: Final = { "tool_choice": "required", "parallel_tool_calls": "true", @@ -3446,8 +3464,8 @@ class TestMalformedParamsFollowDropParams: [("mode", "turbo"), ("context", "forever"), ("summary", "sometimes"), ("generate_summary", "auto")], ) def test_a_reasoning_value_the_model_row_does_not_declare_is_dropped_under_drop_params( - self, local_model_cost_map, monkeypatch, field, value - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, field: str, value: str + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) row: Final = litellm.model_cost["gpt-6.1-sol"] declared: Final = { @@ -3466,8 +3484,8 @@ class TestMalformedParamsFollowDropParams: @pytest.mark.parametrize(("field", "declarations"), [("mode", "reasoning_modes"), ("context", "reasoning_contexts")]) def test_a_row_without_reasoning_declarations_drops_values_no_openai_row_declares( - self, local_model_cost_map, monkeypatch, field, declarations - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, field: str, declarations: str + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) undeclared_model: Final = next( name @@ -3487,7 +3505,9 @@ class TestMalformedParamsFollowDropParams: assert sent["reasoning"] == malformed assert kept["reasoning"] == declared - def test_reasoning_on_a_non_reasoning_model_is_dropped_under_drop_params(self, local_model_cost_map, monkeypatch): + def test_reasoning_on_a_non_reasoning_model_is_dropped_under_drop_params( + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) non_reasoning_model: Final = next( name @@ -3515,8 +3535,8 @@ class TestMalformedParamsFollowDropParams: ids=["openai", "azure", "azure-o-series"], ) def test_an_effort_outside_the_known_levels_is_dropped_unless_the_deployment_declares_it( - self, local_model_cost_map, monkeypatch, config, model - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, config: OpenAIResponsesAPIConfig, model: str + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) reasoning: Final = {"effort": "not-an-effort", "summary": "auto"} @@ -3543,8 +3563,8 @@ class TestMalformedParamsFollowDropParams: ids=["azure", "azure-o-series"], ) def test_azure_drops_reasoning_values_and_keys_no_openai_row_declares( - self, local_model_cost_map, monkeypatch, config, model - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, config: OpenAIResponsesAPIConfig, model: str + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) row: Final = litellm.model_cost["gpt-6.1-sol"] declared: Final = { @@ -3570,8 +3590,8 @@ class TestMalformedParamsFollowDropParams: assert sent["reasoning"] == malformed def test_an_openai_model_outside_the_catalog_keeps_reasoning_keys_openai_does_not_define( - self, local_model_cost_map, monkeypatch - ): + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setattr(litellm, "drop_params", False) server_keys: Final = {"max_tokens": 2000, "exclude": True} requested: Final = {"effort": "high", "mode": "turbo", "context": "forever", "summary": "sometimes", **server_keys} @@ -3582,7 +3602,7 @@ class TestMalformedParamsFollowDropParams: assert dropped["reasoning"] == {"effort": "high", **server_keys} - def test_a_reasoning_mode_a_reloaded_cost_map_declares_is_kept_on_azure(self, local_model_cost_map): + def test_a_reasoning_mode_a_reloaded_cost_map_declares_is_kept_on_azure(self, local_model_cost_map: None) -> None: row: Final = litellm.model_cost["gpt-6.1-sol"] new_mode: Final = "-".join(row["reasoning_modes"]) + "-next" request: Final = {"reasoning": {"effort": "high", "mode": new_mode}} diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index d32fcfd47ea..0b062c4c75b 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -1014,8 +1014,8 @@ def test_responses_object_bills_an_explicit_cache_hit_at_the_explicit_rate(): @pytest.mark.parametrize(("field", "declarations"), [("mode", "reasoning_modes"), ("context", "reasoning_contexts")]) def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_under_drop_params( - local_model_cost_map, field, declarations -): + local_model_cost_map: None, field: str, declarations: str +) -> None: undeclared_model: Final = next( name for name, row in litellm.model_cost.items() @@ -1044,8 +1044,8 @@ def test_the_chat_bridge_drops_a_reasoning_value_no_openai_row_declares_under_dr assert bridged(False) == requested def test_the_chat_bridge_outside_the_openai_catalog_drops_undeclared_values_and_keeps_other_keys( - local_model_cost_map, -): + local_model_cost_map: None, +) -> None: declared_mode: Final = litellm.model_cost["gpt-6.1-sol"]["reasoning_modes"][-1] requested: Final = {"effort": "high", "mode": declared_mode, "context": "forever", "max_tokens": 2000} @@ -1068,8 +1068,8 @@ def test_the_chat_bridge_outside_the_openai_catalog_drops_undeclared_values_and_ def test_the_chat_bridge_keeps_other_keys_for_an_openai_prefixed_name_it_resolves_to_a_catalog_row( - local_model_cost_map, -): + local_model_cost_map: None, +) -> None: declared_mode: Final = litellm.model_cost["gpt-6.1-sol"]["reasoning_modes"][-1] requested: Final = {"effort": "high", "mode": "not-a-declared-value", "max_tokens": 2000} @@ -1092,7 +1092,7 @@ def test_the_chat_bridge_keeps_other_keys_for_an_openai_prefixed_name_it_resolve assert bridged("gpt-6.1-sol", True) == {"effort": "high", "mode": declared_mode} -def test_the_chat_bridge_keeps_only_effort_for_a_catalog_model_that_does_not_reason(local_model_cost_map): +def test_the_chat_bridge_keeps_only_effort_for_a_catalog_model_that_does_not_reason(local_model_cost_map: None) -> None: non_reasoning_model: Final = next( name for name, row in litellm.model_cost.items()