From 65d886be30de635196182f4293edc7846a374f5d Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 29 Oct 2026 07:49:58 +0000 Subject: [PATCH 1/8] fix(tool_choice): accept OpenAI allowed_tools on chat completions and flatten it for the Responses bridge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../transformation.py | 17 +- .../allowed_tools_tool_choice.py | 74 ++++ litellm/main.py | 18 +- litellm/utils.py | 15 + ...responses_transformation_transformation.py | 53 ++- .../test_allowed_tools_tool_choice.py | 89 +++++ tests/unit/test_main.py | 377 +++++++++--------- tests/unit/test_utils.py | 79 +++- 8 files changed, 504 insertions(+), 218 deletions(-) create mode 100644 litellm/litellm_core_utils/allowed_tools_tool_choice.py create mode 100644 tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index ee3639ac387..b08426f0127 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -16,6 +16,7 @@ from openai.types.responses.response_input_param import ( ResponseCustomToolCallOutputParam, ResponseCustomToolCallParam, ) +from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam @@ -25,6 +26,10 @@ import litellm from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + parse_chat_allowed_tools_tool_choice, + responses_allowed_tools_tool_choice, +) from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( responses_reasoning_items_from_thinking_blocks, @@ -732,13 +737,23 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def _normalize_tool_choice_for_responses_api( self, tool_choice: _ToolChoiceT - ) -> _ToolChoiceT | ToolChoiceFunctionParam | ToolChoiceCustomParam | Literal["auto", "none", "required"]: + ) -> ( + _ToolChoiceT + | ToolChoiceFunctionParam + | ToolChoiceCustomParam + | ToolChoiceAllowedParam + | Literal["auto", "none", "required"] + ): """Chat tool_choice nests the name under function/custom; Responses API expects top-level name.""" if not isinstance(tool_choice, dict): return tool_choice choice_type: Final = tool_choice.get("type") if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"): return choice_type + if choice_type == "allowed_tools": + parsed: Final = parse_chat_allowed_tools_tool_choice(tool_choice) + flat: Final = responses_allowed_tools_tool_choice(parsed) if parsed is not None else None + return tool_choice if flat is None else flat if choice_type not in ("function", "custom"): return tool_choice if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"): diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py new file mode 100644 index 00000000000..f31f9a3c12b --- /dev/null +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -0,0 +1,74 @@ +from collections.abc import Mapping +from typing import Final, Literal + +from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError + +ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"}) + +CHAT_ALLOWED_TOOLS_LAYOUT: Final = ( + '{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": []}}' +) + + +class ChatAllowedTools(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + mode: Literal["auto", "required"] + tools: tuple[Mapping[str, object], ...] = Field(min_length=1) + + +class ChatAllowedToolsToolChoice(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + type: Literal["allowed_tools"] + allowed_tools: ChatAllowedTools + + +_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice) + + +class _NamedTool(BaseModel): + model_config = ConfigDict(frozen=True) + + name: str = Field(min_length=1) + + +_NAMED_TOOL: Final = TypeAdapter(_NamedTool) + + +def is_allowed_tools_tool_choice(tool_choice: Mapping[str, object] | str | None) -> bool: + return isinstance(tool_choice, Mapping) and tool_choice.get("type") == "allowed_tools" + + +def parse_chat_allowed_tools_tool_choice(tool_choice: Mapping[str, object]) -> ChatAllowedToolsToolChoice | None: + try: + return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) + except ValidationError: + return None + + +def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None: + tool_type: Final = tool.get("type") + if tool_type not in ("function", "custom"): + return tool + if isinstance(tool.get("name"), str): + return tool + try: + nested: Final = _NAMED_TOOL.validate_python(tool.get(tool_type)) + except ValidationError: + return None + return {"type": tool_type, "name": nested.name} + + +def responses_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ToolChoiceAllowedParam | None: + """The flat Responses API layout of a chat ``allowed_tools`` choice, or ``None`` when an entry is not one the + chat layout defines, so the caller sends the original rather than a rewrite that could mean something looser.""" + flattened: Final = tuple(_responses_allowed_tool(tool) for tool in choice.allowed_tools.tools) + if any(tool is None for tool in flattened): + return None + return ToolChoiceAllowedParam( + type="allowed_tools", + mode=choice.allowed_tools.mode, + tools=[dict(tool) for tool in flattened if tool is not None], + ) diff --git a/litellm/main.py b/litellm/main.py index 41c4302499e..015c0c857ca 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -67,8 +67,12 @@ from litellm.constants import ( NADIR_DEFAULT_API_BASE, OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS, ) -from litellm.exceptions import LiteLLMUnknownProvider +from litellm.exceptions import BadRequestError, LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS, + is_allowed_tools_tool_choice, +) from litellm.litellm_core_utils.asyncify import asyncify, run_async_function from litellm.litellm_core_utils.audio_utils.utils import ( calculate_request_duration, @@ -5752,6 +5756,18 @@ def completion( if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): tools = _drop_anthropic_only_tool_keys(tools=tools) + if is_allowed_tools_tool_choice(tool_choice) and custom_llm_provider not in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: + raise BadRequestError( + message=( + "tool_choice type 'allowed_tools' is only supported for " + f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. " + "drop_params does not drop it, since that would let the model call tools the request excluded. " + "Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead." + ), + model=model, + llm_provider=custom_llm_provider, + ) + if provider_specific_header is not None: headers.update( ProviderSpecificHeaderUtils.get_provider_specific_headers( diff --git a/litellm/utils.py b/litellm/utils.py index 72c37c3760e..b58db3a7938 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -82,6 +82,10 @@ from litellm.constants import ( PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + CHAT_ALLOWED_TOOLS_LAYOUT, + parse_chat_allowed_tools_tool_choice, +) from litellm.litellm_core_utils.core_helpers import ( bind_budget_reservation_to_callbacks, normalize_drop_params, @@ -9064,6 +9068,17 @@ def validate_chat_completion_tool_choice( tool_choice_type = tool_choice.get("type") if tool_choice_type in ("auto", "none", "required") and "function" not in tool_choice: return tool_choice_type + if tool_choice_type == "allowed_tools": + if parse_chat_allowed_tools_tool_choice(tool_choice) is None: + raise BadRequestError( + message=( + f"Invalid tool choice, tool_choice={tool_choice}. On chat completions allowed_tools is " + f"{CHAT_ALLOWED_TOOLS_LAYOUT}; the flat Responses API layout is not accepted here" + ), + model=model, + llm_provider="", + ) + return tool_choice # Standard OpenAI format: {"type": "function", "function": {...}} if tool_choice.get("type") is None or tool_choice.get("function") is None: diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 46f1dbf2e7a..0f1aa6442f2 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1270,9 +1270,7 @@ def test_transform_request_forwards_defer_loading_only_beside_tool_search( [{"type": "tool_search", "execution": "client"}, {"type": "tool_search"}], ids=["client-only", "hosted-and-client"], ) -def test_transform_request_drops_defer_loading_beside_client_tool_search( - deferred_tool, drop_params, tool_search_tool -): +def test_transform_request_drops_defer_loading_beside_client_tool_search(deferred_tool, drop_params, tool_search_tool): tool_search_tools: Final = ( [tool_search_tool] if tool_search_tool.get("execution") == "client" @@ -2992,6 +2990,38 @@ def test_map_optional_params_tool_choice_chat_nested_to_responses_api(): {"type": "custom", "name": "ApplyPatch"}, ), ({"type": "custom"}, {"type": "custom"}), + ( + { + "type": "allowed_tools", + "allowed_tools": { + "mode": "required", + "tools": [ + {"type": "function", "function": {"name": "get_time"}}, + {"type": "custom", "custom": {"name": "apply_patch"}}, + {"type": "mcp", "server_label": "docs"}, + {"type": "function", "name": "already_flat"}, + ], + }, + }, + { + "type": "allowed_tools", + "mode": "required", + "tools": [ + {"type": "function", "name": "get_time"}, + {"type": "custom", "name": "apply_patch"}, + {"type": "mcp", "server_label": "docs"}, + {"type": "function", "name": "already_flat"}, + ], + }, + ), + ( + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}}, + ), + ( + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + ), ], ) def test_normalize_tool_choice_for_responses_api(tool_choice, expected): @@ -4033,6 +4063,16 @@ def test_output_item_done_with_stream_map_keeps_empty_delta(): ({"type": "required"}, "required"), ("auto", "auto"), ({"type": "function", "function": {"name": "get_weather"}}, {"type": "function", "name": "get_weather"}), + ( + { + "type": "allowed_tools", + "allowed_tools": { + "mode": "required", + "tools": [{"type": "function", "function": {"name": "get_weather"}}], + }, + }, + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]}, + ), ], ) async def test_acompletion_bridge_normalizes_tool_choice_on_the_wire( @@ -4105,6 +4145,7 @@ async def test_acompletion_bridge_normalizes_tool_choice_on_the_wire( post_kwargs = mock_post.call_args.kwargs request_body = post_kwargs["json"] if "json" in post_kwargs else json.loads(post_kwargs["data"]) assert request_body["tool_choice"] == expected_wire_tool_choice + assert [tool["name"] for tool in request_body["tools"]] == ["get_weather"] def _make_incomplete_responses_api_response( @@ -4396,7 +4437,11 @@ def test_tool_search_items_replay_only_to_a_request_that_carries_tool_search(too "provider_specific_fields": { "tool_search_items": [ {"type": "tool_search_call", "id": "tsc_1", "arguments": {"paths": ["get_weather"]}}, - {"type": "tool_search_output", "id": "tso_1", "tools": [{"type": "function", "name": "get_weather"}]}, + { + "type": "tool_search_output", + "id": "tso_1", + "tools": [{"type": "function", "name": "get_weather"}], + }, ] }, }, diff --git a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py new file mode 100644 index 00000000000..4fea90b091f --- /dev/null +++ b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py @@ -0,0 +1,89 @@ +import copy +from typing import Final + +import pytest + +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + ChatAllowedToolsToolChoice, + is_allowed_tools_tool_choice, + parse_chat_allowed_tools_tool_choice, + responses_allowed_tools_tool_choice, +) + +_CHAT_LAYOUT: Final = { + "type": "allowed_tools", + "allowed_tools": { + "mode": "auto", + "tools": [ + {"type": "function", "function": {"name": "get_time"}}, + {"type": "custom", "custom": {"name": "apply_patch"}}, + {"type": "mcp", "server_label": "docs", "server_url": "https://docs.example"}, + {"type": "function", "name": "already_flat"}, + ], + }, +} + + +def _parsed(tool_choice: dict[str, object]) -> ChatAllowedToolsToolChoice: + parsed: Final = parse_chat_allowed_tools_tool_choice(tool_choice) + assert parsed is not None + return parsed + + +def test_responses_layout_lifts_mode_and_tools_and_flattens_names_in_order(): + assert responses_allowed_tools_tool_choice(_parsed(_CHAT_LAYOUT)) == { + "type": "allowed_tools", + "mode": "auto", + "tools": [ + {"type": "function", "name": "get_time"}, + {"type": "custom", "name": "apply_patch"}, + {"type": "mcp", "server_label": "docs", "server_url": "https://docs.example"}, + {"type": "function", "name": "already_flat"}, + ], + } + + +def test_responses_layout_leaves_the_caller_dict_untouched(): + original: Final = copy.deepcopy(_CHAT_LAYOUT) + flat: Final = responses_allowed_tools_tool_choice(_parsed(_CHAT_LAYOUT)) + assert flat is not None + flat["tools"][0]["name"] = "mutated" + assert _CHAT_LAYOUT == original + + +@pytest.mark.parametrize( + "tool", + [{"type": "function"}, {"type": "function", "function": {}}, {"type": "custom", "custom": {"name": ""}}], +) +def test_responses_layout_refuses_to_rewrite_an_entry_without_a_name(tool: dict[str, object]): + """A half-rewritten restriction could mean something looser, so the caller sends the original instead.""" + tool_choice: Final = {"type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": [tool]}} + assert responses_allowed_tools_tool_choice(_parsed(tool_choice)) is None + + +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "any", "tools": [{"type": "function"}]}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}, "extra": 1}, + {"type": "function", "function": {"name": "get_time"}}, + ], +) +def test_parse_rejects_anything_but_the_chat_layout(tool_choice: dict[str, object]): + assert parse_chat_allowed_tools_tool_choice(tool_choice) is None + + +@pytest.mark.parametrize( + ("tool_choice", "expected"), + [ + (_CHAT_LAYOUT, True), + ({"type": "allowed_tools"}, True), + ({"type": "function"}, False), + ("auto", False), + (None, False), + ], +) +def test_is_allowed_tools_tool_choice(tool_choice: object, expected: bool): + assert is_allowed_tools_tool_choice(tool_choice) is expected diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index f38bbcf4404..499f53e15a5 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -291,9 +291,7 @@ def test_completion_missing_role(openai_api_response): print(f"openai_api_response: {openai_api_response}") - with patch.object( - client, "post", MagicMock(return_value=mock_raw_response) - ) as mock_create: + with patch.object(client, "post", MagicMock(return_value=mock_raw_response)) as mock_create: litellm.completion( model="gpt-4o-mini", messages=[ @@ -565,6 +563,66 @@ def test_completion_strips_eager_input_streaming_before_openai(respx_mock: respx assert sent_tool["function"]["name"] == "write_file" +_ALLOWED_TOOLS_TOOL_CHOICE: Final = { + "type": "allowed_tools", + "allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, +} +_TWO_TOOLS: Final = [ + {"type": "function", "function": {"name": "get_time", "parameters": {"type": "object", "properties": {}}}}, + {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}}, +] + + +@pytest.mark.parametrize("model", ["openai/gpt-5.5", "azure/my-gpt-deployment"]) +def test_completion_forwards_allowed_tools_tool_choice_to_openai_compatible_chat( + respx_mock: respx.MockRouter, openai_api_response, model: str +): + """OpenAI and Azure enforce allowed_tools themselves, so the chat layout and the full tool list go out as sent.""" + mock_route: Final = respx_mock.post(url__regex=r"http://localhost:12346/.*chat/completions.*").mock( + return_value=httpx.Response(status_code=200, json=openai_api_response) + ) + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "What time is it?"}], + tools=_TWO_TOOLS, + tool_choice=_ALLOWED_TOOLS_TOOL_CHOICE, + api_base="http://localhost:12346/v1", + api_key="fake_api_key", + api_version="2025-04-01-preview", + ) + + assert mock_route.called + sent: Final = json.loads(respx_mock.calls[0].request.content) + assert sent["tool_choice"] == _ALLOWED_TOOLS_TOOL_CHOICE + assert [tool["function"]["name"] for tool in sent["tools"]] == ["get_time", "get_weather"] + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("model", ["anthropic/claude-sonnet-4-5", "gemini/gemini-2.5-flash", "bedrock/converse/x"]) +def test_completion_allowed_tools_tool_choice_is_a_400_where_the_provider_cannot_enforce_it( + respx_mock: respx.MockRouter, model: str, drop_params: bool +): + """A provider adapter that does not know allowed_tools would send no restriction at all, so the call stops with + a 400 before any request leaves, and drop_params does not quietly widen what the model may call.""" + with pytest.raises( + litellm.BadRequestError, match="allowed_tools.*only supported for azure, litellm_proxy, openai" + ) as exc: + litellm.completion( + model=model, + messages=[{"role": "user", "content": "What time is it?"}], + tools=_TWO_TOOLS, + tool_choice=_ALLOWED_TOOLS_TOOL_CHOICE, + drop_params=drop_params, + api_key="fake_api_key", + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + assert exc.value.status_code == 400 + assert not respx_mock.calls + + _CLAUDE_TOOL_SEARCH_CALL: Final = { "id": "srvtoolu_01search", "type": "function", @@ -2860,9 +2918,7 @@ def test_parallel_function_call_stream_reassembles_each_arguments_payload(): created=1751934860, model="gpt-4.1-mini", object="chat.completion.chunk", - choices=[ - StreamingChoices(index=0, delta=Delta(tool_calls=tool_calls), finish_reason=finish_reason) - ], + choices=[StreamingChoices(index=0, delta=Delta(tool_calls=tool_calls), finish_reason=finish_reason)], ) chunks: Final = [ @@ -2993,9 +3049,7 @@ def test_parallel_function_call_stream_over_responses_bridge_ignores_events_it_d }, ) route: Final = respx.post("https://api.openai.com/v1/responses").mock( - return_value=httpx.Response( - 200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events) - ) + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events)) ) response: Final = litellm.completion( @@ -3609,9 +3663,7 @@ async def test_retrying() -> None: ) -def test_anthropic_disable_url_suffix_env_var( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter -) -> None: +def test_anthropic_disable_url_suffix_env_var(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> None: monkeypatch.setenv("ANTHROPIC_API_BASE", "https://api.example.com") default_route: Final = respx_mock.post("https://api.example.com/v1/messages").mock( return_value=httpx.Response( @@ -3874,9 +3926,7 @@ async def test_async_mock_streaming_request(): print(chunk) complete_response += chunk["choices"][0]["delta"]["content"] or "" - assert ( - complete_response == "LiteLLM is awesome" - ), f"Unexpected response got {complete_response}" + assert complete_response == "LiteLLM is awesome", f"Unexpected response got {complete_response}" def test_mock_request_n_greater_than_1(): @@ -4402,7 +4452,10 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai(): assert request_body["prompt_cache_key"] == "test-cache-key" -_TOOL: Final[Mapping[str, object]] = {"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}} +_TOOL: Final[Mapping[str, object]] = { + "type": "function", + "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}, +} # Client settings ride in OPENAI_CHAT_COMPLETION_PARAMS but configure the HTTP client, never the body. _OPENAI_CLIENT_SETTINGS: Final = frozenset( @@ -4462,7 +4515,9 @@ _OPENAI_BODY_PARAM_CASES: Final[Mapping[str, tuple[str, Mapping[str, object]]]] def test_every_openai_body_param_has_a_forwarding_case(): - body_params: Final = frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS + body_params: Final = ( + frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS + ) assert frozenset(_OPENAI_BODY_PARAM_CASES) == body_params, ( "a param added to OPENAI_CHAT_COMPLETION_PARAMS needs a case in _OPENAI_BODY_PARAM_CASES proving " "completion() sends it to OpenAI; prompt_cache_retention (LIT-9758) and store/prompt_cache_key " @@ -5913,12 +5968,8 @@ def test_acompletion_params(): acompletion_params_odict = inspect.signature(acompletion).parameters completion_params_dict = inspect.signature(completion).parameters - acompletion_params = { - name: param.annotation for name, param in acompletion_params_odict.items() - } - completion_params = { - name: param.annotation for name, param in completion_params_dict.items() - } + acompletion_params = {name: param.annotation for name, param in acompletion_params_odict.items()} + completion_params = {name: param.annotation for name, param in completion_params_dict.items()} keys_acompletion = set(acompletion_params.keys()) keys_completion = set(completion_params.keys()) @@ -5972,9 +6023,7 @@ def test_null_role_response(): import openai openai_client = openai.OpenAI() - with patch.object( - openai_client, "post", side_effect=_openai_mock_response - ) as mock_response: + with patch.object(openai_client, "post", side_effect=_openai_mock_response) as mock_response: response = litellm.completion( model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey! how's it going?"}], @@ -6072,7 +6121,8 @@ def test_completion_perplexity_api(): openai_client = OpenAI() with patch.object( - openai_client, "post", + openai_client, + "post", side_effect=_return_pydantic_obj, ) as mock_client: # litellm.set_verbose= True @@ -6115,9 +6165,7 @@ async def test_openai_compatible_custom_api_base(provider): openai_client = OpenAI(api_key="fake-key") - with patch.object( - openai_client, "post", new=MagicMock() - ) as mock_call: + with patch.object(openai_client, "post", new=MagicMock()) as mock_call: try: completion( model="{provider}/my-vllm-model".format(provider=provider), @@ -6166,9 +6214,7 @@ async def test_openai_compatible_custom_api_video(provider): openai_client = OpenAI(api_key="fake-key") - with patch.object( - openai_client, "post", new=MagicMock() - ) as mock_call: + with patch.object(openai_client, "post", new=MagicMock()) as mock_call: try: completion( model="{provider}/my-vllm-model".format(provider=provider), @@ -6361,10 +6407,7 @@ async def test_completion_functions_param(): mock_client.assert_called() print(f"mock_client.call_args.kwargs: {mock_client.call_args.kwargs}") assert "tools" in mock_client.call_args.kwargs["json"] - assert ( - "litellm_param_is_function_call" - not in mock_client.call_args.kwargs["json"] - ) + assert "litellm_param_is_function_call" not in mock_client.call_args.kwargs["json"] assert response.choices[0].message.function_call is not None except Exception as e: pytest.fail(f"Error occurred: {e}") @@ -6440,9 +6483,7 @@ def test_bedrock_deepseek_custom_prompt_dict(): def test_bedrock_deepseek_known_tokenizer_config(monkeypatch): - model = ( - "deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf" - ) + model = "deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf" from unittest.mock import Mock import httpx @@ -6513,9 +6554,7 @@ def test_completion_anthropic_hanging(): {"role": "function", "name": "get_capital", "content": "Kokoko"}, ] - converted_messages = anthropic_messages_pt( - messages, model="claude-3-sonnet-20240229", llm_provider="anthropic" - ) + converted_messages = anthropic_messages_pt(messages, model="claude-3-sonnet-20240229", llm_provider="anthropic") assert len(converted_messages) == 3 for i, msg in enumerate(converted_messages): @@ -6822,9 +6861,7 @@ def test_completion_logprobs_parses_top_logprobs(): "top_logprobs": 3, "reasoning_effort": "none", } - assert [ - item.token for item in response.choices[0].logprobs.content[0].top_logprobs - ] == ["hello", "hi", "hey"] + assert [item.token for item in response.choices[0].logprobs.content[0].top_logprobs] == ["hello", "hi", "hey"] choice: Final = response.choices[0] assert "logprobs" in choice assert "content" in choice.logprobs @@ -6896,9 +6933,11 @@ def test_completion_logprobs_stream_parses_top_logprobs(): "stream_options": {"include_usage": True}, "stream": True, } - assert [ - item.token for item in response_chunks[0].choices[0].logprobs.content[0].top_logprobs - ] == ["hello", "hi", "hey"] + assert [item.token for item in response_chunks[0].choices[0].logprobs.content[0].top_logprobs] == [ + "hello", + "hi", + "hey", + ] @respx.mock @@ -6969,9 +7008,7 @@ async def test_bedrock_converse_parses_scripted_response( "api_key": "test-bedrock-token", "aws_region_name": "us-west-2", } - response: Final = ( - await litellm.acompletion(**request) if async_mode else litellm.completion(**request) - ) + response: Final = await litellm.acompletion(**request) if async_mode else litellm.completion(**request) assert route.calls[0].request.headers["authorization"] == "Bearer test-bedrock-token" assert json.loads(route.calls[0].request.content) == { @@ -7058,9 +7095,7 @@ def test_bedrock_converse_parses_scripted_tool_call_arguments( tool_call: Final = response.choices[0].message.tool_calls[0] assert route.call_count == 1 assert tool_call.function.name == "trade" - assert json.loads(tool_call.function.arguments) == { - "orders": [{"action": "buy", "asset": "BTC", "amount": 0.1}] - } + assert json.loads(tool_call.function.arguments) == {"orders": [{"action": "buy", "asset": "BTC", "amount": 0.1}]} @pytest.mark.parametrize( @@ -7098,9 +7133,9 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( expected_request_body: dict[str, object], fake_provider_credentials: None, ): - route: Final = respx.post( - f"https://bedrock-runtime.us-west-2.amazonaws.com/model/{invoke_model_id}/invoke" - ).mock(return_value=httpx.Response(200, json=response_body)) + route: Final = respx.post(f"https://bedrock-runtime.us-west-2.amazonaws.com/model/{invoke_model_id}/invoke").mock( + return_value=httpx.Response(200, json=response_body) + ) response: Final = litellm.completion( model=model, @@ -7111,9 +7146,7 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( ) request: Final = route.calls[0].request - assert request.headers["authorization"].startswith( - "AWS4-HMAC-SHA256 Credential=unit-test/" - ) + assert request.headers["authorization"].startswith("AWS4-HMAC-SHA256 Credential=unit-test/") assert "/us-west-2/bedrock/aws4_request" in request.headers["authorization"] assert request.headers["x-amz-date"].endswith("Z") assert json.loads(request.content) == expected_request_body @@ -7123,9 +7156,9 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( @respx.mock def test_completion_azure_ad_token_is_forwarded(openai_api_response): - route: Final = respx.post( - "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" - ).mock(return_value=httpx.Response(200, json=openai_api_response)) + route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( + return_value=httpx.Response(200, json=openai_api_response) + ) litellm.completion( model="azure/gpt-6.1-sol", @@ -7140,9 +7173,9 @@ def test_completion_azure_ad_token_is_forwarded(openai_api_response): @respx.mock def test_completion_azure_extra_headers_are_forwarded(openai_api_response): - route: Final = respx.post( - "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" - ).mock(return_value=httpx.Response(200, json=openai_api_response)) + route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( + return_value=httpx.Response(200, json=openai_api_response) + ) litellm.completion( model="azure/gpt-6.1-sol", @@ -7163,9 +7196,9 @@ def test_completion_azure_extra_headers_are_forwarded(openai_api_response): @respx.mock def test_completion_azure_api_key_argument_is_forwarded(openai_api_response): - route: Final = respx.post( - "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" - ).mock(return_value=httpx.Response(200, json=openai_api_response)) + route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( + return_value=httpx.Response(200, json=openai_api_response) + ) response: Final = litellm.completion( model="azure/gpt-6.1-sol", @@ -7822,9 +7855,7 @@ def test_completion_deep_infra(drop_params): ], "usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25}, } - route: Final = respx.post(endpoint).mock( - return_value=httpx.Response(200, json=response_body) - ) + route: Final = respx.post(endpoint).mock(return_value=httpx.Response(200, json=response_body)) if not drop_params: with pytest.raises(litellm.exceptions.UnsupportedParamsError): @@ -7888,9 +7919,7 @@ def test_completion_deep_infra_mistral(): ], "usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25}, } - route: Final = respx.post(endpoint).mock( - return_value=httpx.Response(200, json=response_body) - ) + route: Final = respx.post(endpoint).mock(return_value=httpx.Response(200, json=response_body)) response: Final = completion( model=model_name, @@ -7939,19 +7968,13 @@ def test_unified_auth_params(provider, model, project, region_name, token): translated_optional_params = litellm.utils.get_optional_params(**data) if provider == "azure": - special_auth_params = ( - litellm.AzureOpenAIConfig().get_mapped_special_auth_params() - ) + special_auth_params = litellm.AzureOpenAIConfig().get_mapped_special_auth_params() elif provider == "bedrock": - special_auth_params = ( - litellm.AmazonBedrockGlobalConfig().get_mapped_special_auth_params() - ) + special_auth_params = litellm.AmazonBedrockGlobalConfig().get_mapped_special_auth_params() elif provider == "vertex_ai": special_auth_params = litellm.VertexAIConfig().get_mapped_special_auth_params() elif provider == "watsonx": - special_auth_params = ( - litellm.IBMWatsonXAIConfig().get_mapped_special_auth_params() - ) + special_auth_params = litellm.IBMWatsonXAIConfig().get_mapped_special_auth_params() for param, value in special_auth_params.items(): assert param in data @@ -7983,9 +8006,7 @@ async def test_dynamic_azure_params(stream, sync_mode): mock_client = AsyncMock(return_value="Hello world!") ## CHECK IF CLIENT IS USED (NO PARAM CHANGE) - with patch.object( - client.chat.completions.with_raw_response, "create", new=mock_client - ) as mock_client: + with patch.object(client.chat.completions.with_raw_response, "create", new=mock_client) as mock_client: try: # client.chat.completions.with_raw_response.create = mock_client if sync_mode: @@ -8014,9 +8035,7 @@ async def test_dynamic_azure_params(stream, sync_mode): new_mock_client = AsyncMock(return_value="Hello world!") ## CHECK IF NEW CLIENT IS USED (PARAM CHANGE) - with patch.object( - client.chat.completions.with_raw_response, "create", new=new_mock_client - ) as new_mock_client: + with patch.object(client.chat.completions.with_raw_response, "create", new=new_mock_client) as new_mock_client: try: if sync_mode: _ = completion( @@ -8227,9 +8246,7 @@ def test_completion_novita_ai(): openai_client = OpenAI(api_key="fake-key") - with patch.object( - openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse) - ) as mock_call: + with patch.object(openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse)) as mock_call: mock_call.return_value.headers = {} mock_call.return_value.parse.return_value = litellm.ModelResponse( choices=[{"message": {"role": "assistant", "content": "Hello"}}] @@ -8246,10 +8263,7 @@ def test_completion_novita_ai(): assert response.choices[0].message.content == "Hello" # Verify model is passed correctly - assert ( - mock_call.call_args.kwargs["body"]["model"] - == "meta-llama/llama-3.3-70b-instruct" - ) + assert mock_call.call_args.kwargs["body"]["model"] == "meta-llama/llama-3.3-70b-instruct" # Verify messages are passed correctly assert mock_call.call_args.kwargs["body"]["messages"] == messages @@ -8306,9 +8320,7 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features): litellm.set_verbose = True monkeypatch.setattr(litellm, "enable_preview_features", enable_preview_features) - with patch.object( - client, "post", return_value=MagicMock() - ) as mock_completion: + with patch.object(client, "post", return_value=MagicMock()) as mock_completion: try: resp = litellm.completion( model="openai/gpt-3.5-turbo", @@ -8321,9 +8333,7 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features): mock_completion.assert_called_once() if enable_preview_features: - assert mock_completion.call_args.kwargs["body"]["metadata"] == { - "my-test-key": "my-test-value" - } + assert mock_completion.call_args.kwargs["body"]["metadata"] == {"my-test-key": "my-test-value"} else: assert "metadata" not in mock_completion.call_args.kwargs["body"] @@ -8544,17 +8554,30 @@ async def test_async_responses_still_retries_provider_server_errors(monkeypatch) monkeypatch.setattr(litellm, "num_retries", None) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) with respx.mock as upstream: - response = upstream.post("https://openai-test.invalid/v1/responses").mock(side_effect=[ - httpx.Response(500, json={"error": {"message": "temporary provider failure", "type": "server_error"}}), - httpx.Response(200, json={ - "id": "resp-retry", "object": "response", "created_at": 1, "status": "completed", - "model": "test-model", "output": [], - "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, - }), - ]) + response = upstream.post("https://openai-test.invalid/v1/responses").mock( + side_effect=[ + httpx.Response(500, json={"error": {"message": "temporary provider failure", "type": "server_error"}}), + httpx.Response( + 200, + json={ + "id": "resp-retry", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "test-model", + "output": [], + "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, + }, + ), + ] + ) result = await litellm.aresponses( - model="openai/test-model", input="ping", api_key="test-key", - api_base="https://openai-test.invalid/v1", num_retries=1, max_retries=0, + model="openai/test-model", + input="ping", + api_key="test-key", + api_base="https://openai-test.invalid/v1", + num_retries=1, + max_retries=0, ) assert result.status == "completed" assert response.call_count == 2 @@ -8740,9 +8763,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini", - "choices": [ - {"index": 0, "delta": {}, "finish_reason": "function_call"} - ], + "choices": [{"index": 0, "delta": {}, "finish_reason": "function_call"}], }, ) @@ -8766,9 +8787,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( ), ) ) - route: Final = respx.post(f"{api_base}/chat/completions").mock( - side_effect=lambda _: next(scripted_responses) - ) + route: Final = respx.post(f"{api_base}/chat/completions").mock(side_effect=lambda _: next(scripted_responses)) stream: Final = litellm.completion( model="gpt-4o-mini", api_key="sk-test", @@ -8779,11 +8798,14 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( ) answer_chunks: Final = tuple(stream) - assert "".join( - chunk.choices[0].delta.content - for chunk in answer_chunks - if chunk.choices and chunk.choices[0].delta.content - ) == "The answer is 42." + assert ( + "".join( + chunk.choices[0].delta.content + for chunk in answer_chunks + if chunk.choices and chunk.choices[0].delta.content + ) + == "The answer is 42." + ) assert tuple( chunk.choices[0].finish_reason for chunk in answer_chunks @@ -8800,12 +8822,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( assert answer_chunks[-1].usage.completion_tokens == 2 assert answer_chunks[-1].usage.total_tokens == 6 assert stream._hidden_params["api_base"] == api_base - assert ( - answer_chunks[0]._hidden_params["additional_headers"][ - "llm_provider-x-ratelimit-remaining-requests" - ] - == "17" - ) + assert answer_chunks[0]._hidden_params["additional_headers"]["llm_provider-x-ratelimit-remaining-requests"] == "17" function_stream: Final = litellm.completion( model="gpt-4o-mini", @@ -8817,9 +8834,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( stream=True, ) function_chunks: Final = tuple(function_stream) - assembled: Final = litellm.stream_chunk_builder( - chunks=function_chunks, messages=messages - ) + assembled: Final = litellm.stream_chunk_builder(chunks=function_chunks, messages=messages) function_call: Final = assembled.choices[0].message.function_call assert function_call is not None @@ -8831,12 +8846,8 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( if chunk.choices and chunk.choices[0].finish_reason is not None ) == ("function_call",) assert route.call_count == 2 - assert json.loads(route.calls[0].request.content)["stream_options"] == { - "include_usage": True - } - assert json.loads(route.calls[1].request.content)["function_call"] == { - "name": "get_current_weather" - } + assert json.loads(route.calls[0].request.content)["stream_options"] == {"include_usage": True} + assert json.loads(route.calls[1].request.content)["function_call"] == {"name": "get_current_weather"} assert json.loads(route.calls[1].request.content)["functions"] == list(functions) @@ -8875,15 +8886,9 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, }, ) - body: Final = "".join( - f"data: {json.dumps(event)}\n\n" for event in events - ) + "data: [DONE]\n\n" - route: Final = respx.post( - "https://api.openai.com/v1/chat/completions" - ).mock( - return_value=httpx.Response( - 200, text=body, headers={"content-type": "text/event-stream"} - ) + body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) ) stream: Final = await litellm.acompletion( model="gpt-4o-mini", @@ -8894,11 +8899,10 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( ) chunks: Final = tuple([chunk async for chunk in stream]) - assert "".join( - chunk.choices[0].delta.content - for chunk in chunks - if chunk.choices and chunk.choices[0].delta.content - ) == "async reply" + assert ( + "".join(chunk.choices[0].delta.content for chunk in chunks if chunk.choices and chunk.choices[0].delta.content) + == "async reply" + ) assert tuple(getattr(chunk, "usage", None) for chunk in chunks[:-1]) == ( None, None, @@ -8906,9 +8910,7 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( assert chunks[-1].usage.prompt_tokens == 3 assert chunks[-1].usage.completion_tokens == 2 assert chunks[-1].usage.total_tokens == 5 - assert json.loads(route.calls[0].request.content)["stream_options"] == { - "include_usage": True - } + assert json.loads(route.calls[0].request.content)["stream_options"] == {"include_usage": True} @respx.mock @@ -8945,16 +8947,10 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], }, ) - body: Final = "".join( - f"data: {json.dumps(event)}\n\n" for event in events - ) + "data: [DONE]\n\n" - return httpx.Response( - 200, text=body, headers={"content-type": "text/event-stream"} - ) + body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" + return httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) - route: Final = respx.post( - "https://api.openai.com/v1/chat/completions" - ).mock(side_effect=respond) + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(side_effect=respond) async def collect(prompt: str) -> tuple[str, tuple[str, ...]]: stream = await litellm.acompletion( @@ -8964,9 +8960,7 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( stream=True, ) chunks: Final = tuple([chunk async for chunk in stream]) - response_text: Final = "".join( - chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices - ) + response_text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) finish_reasons: Final = tuple( chunk.choices[0].finish_reason for chunk in chunks @@ -8974,9 +8968,7 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( ) return response_text, finish_reasons - responses: Final = tuple( - await asyncio.gather(collect("request one"), collect("request two")) - ) + responses: Final = tuple(await asyncio.gather(collect("request one"), collect("request two"))) assert tuple(response for response, _ in responses) == ( "response one", @@ -9170,9 +9162,7 @@ def test_streaming_openai_usage_includes_test_owned_cost( "object": "chat.completion.chunk", "created": 1, "model": model_name, - "choices": [ - {"index": 0, "delta": {"role": "assistant", "content": "priced"}, "finish_reason": None} - ], + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "priced"}, "finish_reason": None}], }, { "id": "chatcmpl-cost", @@ -9207,9 +9197,7 @@ def test_streaming_openai_usage_includes_test_owned_cost( ) stream_chunks: Final = tuple(stream) usage_chunks: Final = tuple( - getattr(chunk, "usage", None) - for chunk in stream_chunks - if getattr(chunk, "usage", None) is not None + getattr(chunk, "usage", None) for chunk in stream_chunks if getattr(chunk, "usage", None) is not None ) assert len(usage_chunks) == 1 assert usage_chunks[0].cost == pytest.approx(4 * 0.001 + 2 * 0.002) @@ -9225,9 +9213,7 @@ def test_completion_ensure_alternating_roles_sends_continue_messages(respx_mock: "object": "chat.completion", "created": 0, "model": "databricks-meta-llama-3-1-70b-instruct", - "choices": [ - {"index": 0, "message": {"role": "assistant", "content": "Sure"}, "finish_reason": "stop"} - ], + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Sure"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 40, "completion_tokens": 1, "total_tokens": 41}, }, ) @@ -9437,9 +9423,12 @@ def test_azure_ai_forwards_reasoning_effort_once_the_deployment_declares_levels( assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "adaptive", _KIMI_DEPLOYMENT)["reasoning_effort"] == ( "adaptive" ) - assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "none", {"reasoning_effort_levels": ["low", "high"]})[ - "reasoning_effort" - ] == "none" + assert ( + _chat_body_sent(respx_mock, "azure_ai/my-kimi", "none", {"reasoning_effort_levels": ["low", "high"]})[ + "reasoning_effort" + ] + == "none" + ) def test_a_model_map_declaration_alone_leaves_reasoning_effort_to_drop_params( @@ -9574,7 +9563,9 @@ def test_a_default_effort_beside_a_thinking_budget_sends_no_effort_on_a_declarin thinking={"type": "enabled", "budget_tokens": 2048, **display}, **default_effort, ) - adaptive_alone: Final = body_sent(respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking={"type": "adaptive", **display}) + adaptive_alone: Final = body_sent( + respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking={"type": "adaptive", **display} + ) assert "output_config" not in defaulted assert defaulted == adaptive_alone @@ -10079,7 +10070,9 @@ async def test_aspeech_answers_a_non_string_provider_hint_with_a_bad_request( hint: str, respx_mock: respx.MockRouter ) -> None: with pytest.raises(litellm.BadRequestError) as raised: - await litellm.aspeech(model="kokoro", input="hello", voice="alloy", api_key="fake_openai_api_key", **{hint: 123}) + await litellm.aspeech( + model="kokoro", input="hello", voice="alloy", api_key="fake_openai_api_key", **{hint: 123} + ) assert raised.value.status_code == 400 assert f"Invalid {hint}" in raised.value.message @@ -10269,7 +10262,9 @@ async def test_converse_fallback_prices_the_current_attempt( } if asynchronous: with pytest.raises(litellm.BadRequestError): - await litellm.acompletion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs) + await litellm.acompletion( + model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs + ) else: with pytest.raises(litellm.BadRequestError): litellm.completion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs) @@ -10293,9 +10288,7 @@ async def test_converse_fallback_prices_the_current_attempt( assert result is not None prices: Final = litellm.get_model_info(profile or model, custom_llm_provider="bedrock") expected: Final = ( - 0.001 + 0.002 - if custom_price - else prices["input_cost_per_token"] + prices["output_cost_per_token"] + 0.001 + 0.002 if custom_price else prices["input_cost_per_token"] + prices["output_cost_per_token"] ) active_logging: Final = response.logging_obj if stream else logging_obj assert active_logging.response_cost_calculator(result) == pytest.approx(expected) @@ -10410,7 +10403,9 @@ async def test_bedrock_strict_tools_limit_rejection_is_sent_again_without_strict async def test_bedrock_strict_tools_limit_rejection_reaches_the_caller_without_drop_params( asynchronous: bool, local_model_cost_map: None ) -> None: - sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream="off", drop_params=None) + sent, error = await _converse_with_a_strict_tools_limit_rejection_first( + asynchronous, stream="off", drop_params=None + ) assert error is not None and "Too many strict tools" in str(error) assert len(sent) == 1 diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 665a540e462..9bc3c64de08 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -933,21 +933,11 @@ _COST_MAP_ROW_SCHEMA: Final = { "cache_creation_input_token_cost": {"type": "number"}, "cache_creation_input_token_cost_above_1hr": {"type": "number"}, "cache_creation_input_token_cost_above_12hr": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": { - "type": "number" - }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": {"type": "number"}, "cache_creation_input_token_cost_above_12hr_flex": {"type": "number"}, "cache_creation_input_token_cost_above_12hr_priority": {"type": "number"}, "cache_creation_input_token_cost_above_12hr_batches": {"type": "number"}, @@ -7419,9 +7409,7 @@ async def test_router_resolves_an_oidc_api_key_only_on_a_proxy_level_deployment( try: with respx.mock(assert_all_called=True) as upstream: upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) - await router.acompletion( - model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"} - ) + await router.acompletion(model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"}) sent: Final = upstream.calls.last.request.headers["Authorization"] finally: router.discard() @@ -7489,7 +7477,9 @@ async def test_an_inline_oidc_api_key_reaches_the_endpoint_its_deployment_config with respx.mock(assert_all_called=False) as upstream: upstream.route().respond(200, json=_BEDROCK_REPLY) await router.acompletion( - model="claude", messages=_PING, aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com" + model="claude", + messages=_PING, + aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com", ) sent: Final = tuple( (call.request.url.host, call.request.headers["Authorization"]) for call in upstream.calls @@ -7698,7 +7688,9 @@ async def test_a_team_row_naming_an_oidc_credential_reaches_only_the_credentials ( lambda reference: LiteLLM_Params( **litellm.Router( - model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}}] + model_list=[ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}} + ] ).get_model_list(model_name="openai/*")[0]["litellm_params"] ), lambda reference: "Bearer listing-token", @@ -8247,6 +8239,49 @@ def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice): assert exc_info.value.model == MODEL +_ALLOWED_TOOLS_CHAT_LAYOUT: Final = { + "type": "allowed_tools", + "allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, +} + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +@pytest.mark.parametrize("mode", ["auto", "required"]) +def test_validate_tool_choice_allowed_tools_chat_layout_passes_through(mode): + """OpenAI's chat allowed_tools restriction is forwarded as the caller sent it, not rejected as an unknown dict.""" + tool_choice = { + "type": "allowed_tools", + "allowed_tools": { + "mode": mode, + "tools": [{"type": "function", "function": {"name": "get_time"}}, {"type": "mcp", "server_label": "docs"}], + }, + } + assert validate_chat_completion_tool_choice(tool_choice, model=MODEL) is tool_choice + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]}, + {"type": "allowed_tools"}, + {"type": "allowed_tools", "allowed_tools": {"mode": "required"}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": []}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "none", "tools": [{"type": "function"}]}}, + {"type": "allowed_tools", "allowed_tools": {"tools": [{"type": "function", "function": {"name": "a"}}]}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": "get_time"}}, + ], +) +def test_validate_tool_choice_allowed_tools_malformed_is_a_400(tool_choice): + """The flat Responses layout, a missing or empty tool list, or a mode OpenAI does not define is a 400 that shows + the chat layout, since OpenAI's chat endpoint rejects each of those too.""" + with pytest.raises(litellm.BadRequestError, match="allowed_tools") as exc_info: + validate_chat_completion_tool_choice(tool_choice, model=MODEL) + assert exc_info.value.status_code == 400 + assert exc_info.value.model == MODEL + assert '"mode": "auto" | "required"' in str(exc_info.value) + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("tool_choice", [123, []]) def test_validate_tool_choice_invalid_type_is_a_400(tool_choice): @@ -8670,7 +8705,9 @@ def test_basic_trimming_no_max_tokens_specified(): trimmed_messages = trim_messages(messages, model="gpt-4") print("trimmed messages for gpt-4") print(trimmed_messages) - assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_input_tokens"] + assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"][ + "max_input_tokens" + ] def test_multiple_messages_trimming(): From c8f916119a2da885f6656ab2a0f9044a9dc6166c Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 29 Oct 2026 07:54:26 +0000 Subject: [PATCH 2/8] test: type the allowed_tools tool_choice fixtures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_allowed_tools_tool_choice.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py index 4fea90b091f..ff7fe53363a 100644 --- a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py +++ b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py @@ -1,4 +1,5 @@ import copy +from collections.abc import Mapping from typing import Final import pytest @@ -10,7 +11,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import ( responses_allowed_tools_tool_choice, ) -_CHAT_LAYOUT: Final = { +_CHAT_LAYOUT: Final[dict[str, object]] = { "type": "allowed_tools", "allowed_tools": { "mode": "auto", @@ -24,7 +25,7 @@ _CHAT_LAYOUT: Final = { } -def _parsed(tool_choice: dict[str, object]) -> ChatAllowedToolsToolChoice: +def _parsed(tool_choice: Mapping[str, object]) -> ChatAllowedToolsToolChoice: parsed: Final = parse_chat_allowed_tools_tool_choice(tool_choice) assert parsed is not None return parsed @@ -47,7 +48,7 @@ def test_responses_layout_leaves_the_caller_dict_untouched(): original: Final = copy.deepcopy(_CHAT_LAYOUT) flat: Final = responses_allowed_tools_tool_choice(_parsed(_CHAT_LAYOUT)) assert flat is not None - flat["tools"][0]["name"] = "mutated" + next(iter(flat["tools"]))["name"] = "mutated" assert _CHAT_LAYOUT == original @@ -57,7 +58,10 @@ def test_responses_layout_leaves_the_caller_dict_untouched(): ) def test_responses_layout_refuses_to_rewrite_an_entry_without_a_name(tool: dict[str, object]): """A half-rewritten restriction could mean something looser, so the caller sends the original instead.""" - tool_choice: Final = {"type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": [tool]}} + tool_choice: Final[dict[str, object]] = { + "type": "allowed_tools", + "allowed_tools": {"mode": "required", "tools": [tool]}, + } assert responses_allowed_tools_tool_choice(_parsed(tool_choice)) is None @@ -85,5 +89,5 @@ def test_parse_rejects_anything_but_the_chat_layout(tool_choice: dict[str, objec (None, False), ], ) -def test_is_allowed_tools_tool_choice(tool_choice: object, expected: bool): +def test_is_allowed_tools_tool_choice(tool_choice: Mapping[str, object] | str | None, expected: bool): assert is_allowed_tools_tool_choice(tool_choice) is expected From 0f93308c86ac976098ff083e711cba38c561b029 Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 29 Oct 2026 08:00:39 +0000 Subject: [PATCH 3/8] test: raw regex for the allowed_tools provider gate match Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/test_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 499f53e15a5..dae39f32291 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -606,7 +606,7 @@ def test_completion_allowed_tools_tool_choice_is_a_400_where_the_provider_cannot """A provider adapter that does not know allowed_tools would send no restriction at all, so the call stops with a 400 before any request leaves, and drop_params does not quietly widen what the model may call.""" with pytest.raises( - litellm.BadRequestError, match="allowed_tools.*only supported for azure, litellm_proxy, openai" + litellm.BadRequestError, match=r"allowed_tools.*only supported for azure, litellm_proxy, openai" ) as exc: litellm.completion( model=model, From 47d798f16364ce310fb1e98db361899282a13daf Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 29 Oct 2026 08:27:51 +0000 Subject: [PATCH 4/8] fix(tool_choice): move the allowed_tools provider gate out of completion() so pyright can still analyze it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../allowed_tools_tool_choice.py | 23 +++++++++++++++++++ litellm/main.py | 19 ++++----------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py index f31f9a3c12b..e9e4d08412f 100644 --- a/litellm/litellm_core_utils/allowed_tools_tool_choice.py +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -4,6 +4,8 @@ from typing import Final, Literal from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError +from litellm.exceptions import BadRequestError + ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"}) CHAT_ALLOWED_TOOLS_LAYOUT: Final = ( @@ -41,6 +43,27 @@ def is_allowed_tools_tool_choice(tool_choice: Mapping[str, object] | str | None) return isinstance(tool_choice, Mapping) and tool_choice.get("type") == "allowed_tools" +def raise_if_allowed_tools_tool_choice_unsupported( + tool_choice: Mapping[str, object] | str | None, + custom_llm_provider: str, + model: str, +) -> None: + """Providers outside the set would silently send the request with no tool restriction at all, so the call stops + here with a 400. ``drop_params`` is deliberately not consulted: dropping a restriction widens what the model may call.""" + if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: + return + raise BadRequestError( + message=( + "tool_choice type 'allowed_tools' is only supported for " + f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. " + "drop_params does not drop it, since that would let the model call tools the request excluded. " + "Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead." + ), + model=model, + llm_provider=custom_llm_provider, + ) + + def parse_chat_allowed_tools_tool_choice(tool_choice: Mapping[str, object]) -> ChatAllowedToolsToolChoice | None: try: return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) diff --git a/litellm/main.py b/litellm/main.py index 2d841153dee..0cc670ffd28 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -67,11 +67,10 @@ from litellm.constants import ( NADIR_DEFAULT_API_BASE, OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS, ) -from litellm.exceptions import BadRequestError, LiteLLMUnknownProvider +from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.allowed_tools_tool_choice import ( - ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS, - is_allowed_tools_tool_choice, + raise_if_allowed_tools_tool_choice_unsupported, ) from litellm.litellm_core_utils.asyncify import asyncify, run_async_function from litellm.litellm_core_utils.audio_utils.utils import ( @@ -5759,17 +5758,9 @@ def completion( if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): tools = _drop_anthropic_only_tool_keys(tools=tools) - if is_allowed_tools_tool_choice(tool_choice) and custom_llm_provider not in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: - raise BadRequestError( - message=( - "tool_choice type 'allowed_tools' is only supported for " - f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. " - "drop_params does not drop it, since that would let the model call tools the request excluded. " - "Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead." - ), - model=model, - llm_provider=custom_llm_provider, - ) + raise_if_allowed_tools_tool_choice_unsupported( + tool_choice=tool_choice, custom_llm_provider=custom_llm_provider, model=model + ) if provider_specific_header is not None: headers.update( From e4cc5832450391dd76339e66a4040b6a9ca9c0c4 Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 29 Oct 2026 08:40:01 +0000 Subject: [PATCH 5/8] fix(tool_choice): keep the allowed_tools helpers typed so the basedpyright gate stays level Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../transformation.py | 5 ++-- .../allowed_tools_tool_choice.py | 20 ++++++++++++--- litellm/main.py | 2 +- litellm/utils.py | 25 ++++++++++--------- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 0b091432fb3..32e9389cc2a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -807,9 +807,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"): return choice_type if choice_type == "allowed_tools": - parsed: Final = parse_chat_allowed_tools_tool_choice(tool_choice) + parsed: Final = parse_chat_allowed_tools_tool_choice(cast(object, tool_choice)) flat: Final = responses_allowed_tools_tool_choice(parsed) if parsed is not None else None - return tool_choice if flat is None else flat + if flat is not None: + return flat if choice_type not in ("function", "custom"): return tool_choice if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"): diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py index e9e4d08412f..8fc3b7c5535 100644 --- a/litellm/litellm_core_utils/allowed_tools_tool_choice.py +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -39,12 +39,24 @@ class _NamedTool(BaseModel): _NAMED_TOOL: Final = TypeAdapter(_NamedTool) -def is_allowed_tools_tool_choice(tool_choice: Mapping[str, object] | str | None) -> bool: - return isinstance(tool_choice, Mapping) and tool_choice.get("type") == "allowed_tools" +class _ToolChoiceType(BaseModel): + model_config = ConfigDict(frozen=True) + + type: str + + +_TOOL_CHOICE_TYPE: Final = TypeAdapter(_ToolChoiceType) + + +def is_allowed_tools_tool_choice(tool_choice: object) -> bool: + try: + return _TOOL_CHOICE_TYPE.validate_python(tool_choice).type == "allowed_tools" + except ValidationError: + return False def raise_if_allowed_tools_tool_choice_unsupported( - tool_choice: Mapping[str, object] | str | None, + tool_choice: object, custom_llm_provider: str, model: str, ) -> None: @@ -64,7 +76,7 @@ def raise_if_allowed_tools_tool_choice_unsupported( ) -def parse_chat_allowed_tools_tool_choice(tool_choice: Mapping[str, object]) -> ChatAllowedToolsToolChoice | None: +def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None: try: return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) except ValidationError: diff --git a/litellm/main.py b/litellm/main.py index 0cc670ffd28..3591b42bb1e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5759,7 +5759,7 @@ def completion( tools = _drop_anthropic_only_tool_keys(tools=tools) raise_if_allowed_tools_tool_choice_unsupported( - tool_choice=tool_choice, custom_llm_provider=custom_llm_provider, model=model + tool_choice=cast(object, tool_choice), custom_llm_provider=custom_llm_provider, model=model ) if provider_specific_header is not None: diff --git a/litellm/utils.py b/litellm/utils.py index 2d64fdd59ad..d4bbe5b17e7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9069,20 +9069,21 @@ def validate_chat_completion_tool_choice( tool_choice_type = tool_choice.get("type") if tool_choice_type in ("auto", "none", "required") and "function" not in tool_choice: return tool_choice_type - if tool_choice_type == "allowed_tools": - if parse_chat_allowed_tools_tool_choice(tool_choice) is None: - raise BadRequestError( - message=( - f"Invalid tool choice, tool_choice={tool_choice}. On chat completions allowed_tools is " - f"{CHAT_ALLOWED_TOOLS_LAYOUT}; the flat Responses API layout is not accepted here" - ), - model=model, - llm_provider="", - ) - return tool_choice + if ( + tool_choice_type == "allowed_tools" + and parse_chat_allowed_tools_tool_choice(cast(object, tool_choice)) is None + ): + raise BadRequestError( + message=( + f"Invalid tool choice, tool_choice={tool_choice}. On chat completions allowed_tools is " + f"{CHAT_ALLOWED_TOOLS_LAYOUT}; the flat Responses API layout is not accepted here" + ), + model=model, + llm_provider="", + ) # Standard OpenAI format: {"type": "function", "function": {...}} - if tool_choice.get("type") is None or tool_choice.get("function") is None: + if tool_choice_type != "allowed_tools" and (tool_choice_type is None or tool_choice.get("function") is None): raise BadRequestError( message=f"Invalid tool choice, tool_choice={tool_choice}. Please ensure tool_choice follows the OpenAI spec", model=model, From 13275c82ae444b3bbe12583898f7355e4fa5ddaf Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 29 Oct 2026 08:46:41 +0000 Subject: [PATCH 6/8] fix(tool_choice): type the allowed_tools validation boundary with Any instead of casts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_responses_transformation/transformation.py | 2 +- litellm/litellm_core_utils/allowed_tools_tool_choice.py | 8 ++++---- litellm/main.py | 2 +- litellm/utils.py | 5 +---- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 32e9389cc2a..4766782bd9e 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -807,7 +807,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"): return choice_type if choice_type == "allowed_tools": - parsed: Final = parse_chat_allowed_tools_tool_choice(cast(object, tool_choice)) + parsed: Final = parse_chat_allowed_tools_tool_choice(tool_choice) flat: Final = responses_allowed_tools_tool_choice(parsed) if parsed is not None else None if flat is not None: return flat diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py index 8fc3b7c5535..5b7a8adb2f3 100644 --- a/litellm/litellm_core_utils/allowed_tools_tool_choice.py +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -1,5 +1,5 @@ from collections.abc import Mapping -from typing import Final, Literal +from typing import Any, Final, Literal from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError @@ -48,7 +48,7 @@ class _ToolChoiceType(BaseModel): _TOOL_CHOICE_TYPE: Final = TypeAdapter(_ToolChoiceType) -def is_allowed_tools_tool_choice(tool_choice: object) -> bool: +def is_allowed_tools_tool_choice(tool_choice: Any) -> bool: try: return _TOOL_CHOICE_TYPE.validate_python(tool_choice).type == "allowed_tools" except ValidationError: @@ -56,7 +56,7 @@ def is_allowed_tools_tool_choice(tool_choice: object) -> bool: def raise_if_allowed_tools_tool_choice_unsupported( - tool_choice: object, + tool_choice: Any, custom_llm_provider: str, model: str, ) -> None: @@ -76,7 +76,7 @@ def raise_if_allowed_tools_tool_choice_unsupported( ) -def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None: +def parse_chat_allowed_tools_tool_choice(tool_choice: Any) -> ChatAllowedToolsToolChoice | None: try: return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) except ValidationError: diff --git a/litellm/main.py b/litellm/main.py index 3591b42bb1e..0cc670ffd28 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5759,7 +5759,7 @@ def completion( tools = _drop_anthropic_only_tool_keys(tools=tools) raise_if_allowed_tools_tool_choice_unsupported( - tool_choice=cast(object, tool_choice), custom_llm_provider=custom_llm_provider, model=model + tool_choice=tool_choice, custom_llm_provider=custom_llm_provider, model=model ) if provider_specific_header is not None: diff --git a/litellm/utils.py b/litellm/utils.py index d4bbe5b17e7..1498b30db11 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9069,10 +9069,7 @@ def validate_chat_completion_tool_choice( tool_choice_type = tool_choice.get("type") if tool_choice_type in ("auto", "none", "required") and "function" not in tool_choice: return tool_choice_type - if ( - tool_choice_type == "allowed_tools" - and parse_chat_allowed_tools_tool_choice(cast(object, tool_choice)) is None - ): + if tool_choice_type == "allowed_tools" and parse_chat_allowed_tools_tool_choice(tool_choice) is None: raise BadRequestError( message=( f"Invalid tool choice, tool_choice={tool_choice}. On chat completions allowed_tools is " From 7cd71231c58ac7a06f29b76434f33a2eff54e9c2 Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 29 Oct 2026 08:52:56 +0000 Subject: [PATCH 7/8] fix(tool_choice): widen tool_choice to object at the allowed_tools helpers with a cast-ok reason Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../transformation.py | 6 +++++- .../litellm_core_utils/allowed_tools_tool_choice.py | 8 ++++---- litellm/main.py | 6 +++++- litellm/utils.py | 12 +++++++++++- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 4766782bd9e..53684f94c70 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -807,7 +807,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"): return choice_type if choice_type == "allowed_tools": - parsed: Final = parse_chat_allowed_tools_tool_choice(tool_choice) + parsed: Final = parse_chat_allowed_tools_tool_choice( + cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic + object, tool_choice + ) + ) flat: Final = responses_allowed_tools_tool_choice(parsed) if parsed is not None else None if flat is not None: return flat diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py index 5b7a8adb2f3..8fc3b7c5535 100644 --- a/litellm/litellm_core_utils/allowed_tools_tool_choice.py +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -1,5 +1,5 @@ from collections.abc import Mapping -from typing import Any, Final, Literal +from typing import Final, Literal from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError @@ -48,7 +48,7 @@ class _ToolChoiceType(BaseModel): _TOOL_CHOICE_TYPE: Final = TypeAdapter(_ToolChoiceType) -def is_allowed_tools_tool_choice(tool_choice: Any) -> bool: +def is_allowed_tools_tool_choice(tool_choice: object) -> bool: try: return _TOOL_CHOICE_TYPE.validate_python(tool_choice).type == "allowed_tools" except ValidationError: @@ -56,7 +56,7 @@ def is_allowed_tools_tool_choice(tool_choice: Any) -> bool: def raise_if_allowed_tools_tool_choice_unsupported( - tool_choice: Any, + tool_choice: object, custom_llm_provider: str, model: str, ) -> None: @@ -76,7 +76,7 @@ def raise_if_allowed_tools_tool_choice_unsupported( ) -def parse_chat_allowed_tools_tool_choice(tool_choice: Any) -> ChatAllowedToolsToolChoice | None: +def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None: try: return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) except ValidationError: diff --git a/litellm/main.py b/litellm/main.py index 0cc670ffd28..da7008943af 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5759,7 +5759,11 @@ def completion( tools = _drop_anthropic_only_tool_keys(tools=tools) raise_if_allowed_tools_tool_choice_unsupported( - tool_choice=tool_choice, custom_llm_provider=custom_llm_provider, model=model + tool_choice=cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic + object, tool_choice + ), + custom_llm_provider=custom_llm_provider, + model=model, ) if provider_specific_header is not None: diff --git a/litellm/utils.py b/litellm/utils.py index 1498b30db11..96b4c6a323c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9069,7 +9069,17 @@ def validate_chat_completion_tool_choice( tool_choice_type = tool_choice.get("type") if tool_choice_type in ("auto", "none", "required") and "function" not in tool_choice: return tool_choice_type - if tool_choice_type == "allowed_tools" and parse_chat_allowed_tools_tool_choice(tool_choice) is None: + if ( + tool_choice_type == "allowed_tools" + and ( + parse_chat_allowed_tools_tool_choice( + cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic + object, tool_choice + ) + ) + is None + ) + ): raise BadRequestError( message=( f"Invalid tool choice, tool_choice={tool_choice}. On chat completions allowed_tools is " From 882330f234a1a76ebce4355fcbb75e654ffb8302 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 29 Oct 2026 12:22:11 -0700 Subject: [PATCH 8/8] test(tool_choice): drop allowed_tools test docstrings, type the new tests, and revert unrelated reformatting Also reword the allowed_tools 400 so it no longer blames the flat Responses layout for every malformed choice --- litellm/utils.py | 4 +- .../test_allowed_tools_tool_choice.py | 1 - tests/unit/test_main.py | 322 +++++++++++------- tests/unit/test_utils.py | 51 ++- 4 files changed, 218 insertions(+), 160 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 96b4c6a323c..7ee7936848c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9082,8 +9082,8 @@ def validate_chat_completion_tool_choice( ): raise BadRequestError( message=( - f"Invalid tool choice, tool_choice={tool_choice}. On chat completions allowed_tools is " - f"{CHAT_ALLOWED_TOOLS_LAYOUT}; the flat Responses API layout is not accepted here" + f"Invalid tool choice, tool_choice={tool_choice}. On chat completions an allowed_tools " + f"tool_choice must be {CHAT_ALLOWED_TOOLS_LAYOUT}" ), model=model, llm_provider="", diff --git a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py index ff7fe53363a..9b7b7eac5e2 100644 --- a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py +++ b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py @@ -57,7 +57,6 @@ def test_responses_layout_leaves_the_caller_dict_untouched(): [{"type": "function"}, {"type": "function", "function": {}}, {"type": "custom", "custom": {"name": ""}}], ) def test_responses_layout_refuses_to_rewrite_an_entry_without_a_name(tool: dict[str, object]): - """A half-rewritten restriction could mean something looser, so the caller sends the original instead.""" tool_choice: Final[dict[str, object]] = { "type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": [tool]}, diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index dae39f32291..fbbc38817d5 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -291,7 +291,9 @@ def test_completion_missing_role(openai_api_response): print(f"openai_api_response: {openai_api_response}") - with patch.object(client, "post", MagicMock(return_value=mock_raw_response)) as mock_create: + with patch.object( + client, "post", MagicMock(return_value=mock_raw_response) + ) as mock_create: litellm.completion( model="gpt-4o-mini", messages=[ @@ -575,9 +577,8 @@ _TWO_TOOLS: Final = [ @pytest.mark.parametrize("model", ["openai/gpt-5.5", "azure/my-gpt-deployment"]) def test_completion_forwards_allowed_tools_tool_choice_to_openai_compatible_chat( - respx_mock: respx.MockRouter, openai_api_response, model: str + respx_mock: respx.MockRouter, openai_api_response: dict[str, object], model: str ): - """OpenAI and Azure enforce allowed_tools themselves, so the chat layout and the full tool list go out as sent.""" mock_route: Final = respx_mock.post(url__regex=r"http://localhost:12346/.*chat/completions.*").mock( return_value=httpx.Response(status_code=200, json=openai_api_response) ) @@ -603,8 +604,6 @@ def test_completion_forwards_allowed_tools_tool_choice_to_openai_compatible_chat def test_completion_allowed_tools_tool_choice_is_a_400_where_the_provider_cannot_enforce_it( respx_mock: respx.MockRouter, model: str, drop_params: bool ): - """A provider adapter that does not know allowed_tools would send no restriction at all, so the call stops with - a 400 before any request leaves, and drop_params does not quietly widen what the model may call.""" with pytest.raises( litellm.BadRequestError, match=r"allowed_tools.*only supported for azure, litellm_proxy, openai" ) as exc: @@ -2918,7 +2917,9 @@ def test_parallel_function_call_stream_reassembles_each_arguments_payload(): created=1751934860, model="gpt-4.1-mini", object="chat.completion.chunk", - choices=[StreamingChoices(index=0, delta=Delta(tool_calls=tool_calls), finish_reason=finish_reason)], + choices=[ + StreamingChoices(index=0, delta=Delta(tool_calls=tool_calls), finish_reason=finish_reason) + ], ) chunks: Final = [ @@ -3049,7 +3050,9 @@ def test_parallel_function_call_stream_over_responses_bridge_ignores_events_it_d }, ) route: Final = respx.post("https://api.openai.com/v1/responses").mock( - return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events)) + return_value=httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events) + ) ) response: Final = litellm.completion( @@ -3663,7 +3666,9 @@ async def test_retrying() -> None: ) -def test_anthropic_disable_url_suffix_env_var(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> None: +def test_anthropic_disable_url_suffix_env_var( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: monkeypatch.setenv("ANTHROPIC_API_BASE", "https://api.example.com") default_route: Final = respx_mock.post("https://api.example.com/v1/messages").mock( return_value=httpx.Response( @@ -3926,7 +3931,9 @@ async def test_async_mock_streaming_request(): print(chunk) complete_response += chunk["choices"][0]["delta"]["content"] or "" - assert complete_response == "LiteLLM is awesome", f"Unexpected response got {complete_response}" + assert ( + complete_response == "LiteLLM is awesome" + ), f"Unexpected response got {complete_response}" def test_mock_request_n_greater_than_1(): @@ -4452,10 +4459,7 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai(): assert request_body["prompt_cache_key"] == "test-cache-key" -_TOOL: Final[Mapping[str, object]] = { - "type": "function", - "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}, -} +_TOOL: Final[Mapping[str, object]] = {"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}} # Client settings ride in OPENAI_CHAT_COMPLETION_PARAMS but configure the HTTP client, never the body. _OPENAI_CLIENT_SETTINGS: Final = frozenset( @@ -4515,9 +4519,7 @@ _OPENAI_BODY_PARAM_CASES: Final[Mapping[str, tuple[str, Mapping[str, object]]]] def test_every_openai_body_param_has_a_forwarding_case(): - body_params: Final = ( - frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS - ) + body_params: Final = frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS assert frozenset(_OPENAI_BODY_PARAM_CASES) == body_params, ( "a param added to OPENAI_CHAT_COMPLETION_PARAMS needs a case in _OPENAI_BODY_PARAM_CASES proving " "completion() sends it to OpenAI; prompt_cache_retention (LIT-9758) and store/prompt_cache_key " @@ -5968,8 +5970,12 @@ def test_acompletion_params(): acompletion_params_odict = inspect.signature(acompletion).parameters completion_params_dict = inspect.signature(completion).parameters - acompletion_params = {name: param.annotation for name, param in acompletion_params_odict.items()} - completion_params = {name: param.annotation for name, param in completion_params_dict.items()} + acompletion_params = { + name: param.annotation for name, param in acompletion_params_odict.items() + } + completion_params = { + name: param.annotation for name, param in completion_params_dict.items() + } keys_acompletion = set(acompletion_params.keys()) keys_completion = set(completion_params.keys()) @@ -6023,7 +6029,9 @@ def test_null_role_response(): import openai openai_client = openai.OpenAI() - with patch.object(openai_client, "post", side_effect=_openai_mock_response) as mock_response: + with patch.object( + openai_client, "post", side_effect=_openai_mock_response + ) as mock_response: response = litellm.completion( model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey! how's it going?"}], @@ -6121,8 +6129,7 @@ def test_completion_perplexity_api(): openai_client = OpenAI() with patch.object( - openai_client, - "post", + openai_client, "post", side_effect=_return_pydantic_obj, ) as mock_client: # litellm.set_verbose= True @@ -6165,7 +6172,9 @@ async def test_openai_compatible_custom_api_base(provider): openai_client = OpenAI(api_key="fake-key") - with patch.object(openai_client, "post", new=MagicMock()) as mock_call: + with patch.object( + openai_client, "post", new=MagicMock() + ) as mock_call: try: completion( model="{provider}/my-vllm-model".format(provider=provider), @@ -6214,7 +6223,9 @@ async def test_openai_compatible_custom_api_video(provider): openai_client = OpenAI(api_key="fake-key") - with patch.object(openai_client, "post", new=MagicMock()) as mock_call: + with patch.object( + openai_client, "post", new=MagicMock() + ) as mock_call: try: completion( model="{provider}/my-vllm-model".format(provider=provider), @@ -6407,7 +6418,10 @@ async def test_completion_functions_param(): mock_client.assert_called() print(f"mock_client.call_args.kwargs: {mock_client.call_args.kwargs}") assert "tools" in mock_client.call_args.kwargs["json"] - assert "litellm_param_is_function_call" not in mock_client.call_args.kwargs["json"] + assert ( + "litellm_param_is_function_call" + not in mock_client.call_args.kwargs["json"] + ) assert response.choices[0].message.function_call is not None except Exception as e: pytest.fail(f"Error occurred: {e}") @@ -6483,7 +6497,9 @@ def test_bedrock_deepseek_custom_prompt_dict(): def test_bedrock_deepseek_known_tokenizer_config(monkeypatch): - model = "deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf" + model = ( + "deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf" + ) from unittest.mock import Mock import httpx @@ -6554,7 +6570,9 @@ def test_completion_anthropic_hanging(): {"role": "function", "name": "get_capital", "content": "Kokoko"}, ] - converted_messages = anthropic_messages_pt(messages, model="claude-3-sonnet-20240229", llm_provider="anthropic") + converted_messages = anthropic_messages_pt( + messages, model="claude-3-sonnet-20240229", llm_provider="anthropic" + ) assert len(converted_messages) == 3 for i, msg in enumerate(converted_messages): @@ -6861,7 +6879,9 @@ def test_completion_logprobs_parses_top_logprobs(): "top_logprobs": 3, "reasoning_effort": "none", } - assert [item.token for item in response.choices[0].logprobs.content[0].top_logprobs] == ["hello", "hi", "hey"] + assert [ + item.token for item in response.choices[0].logprobs.content[0].top_logprobs + ] == ["hello", "hi", "hey"] choice: Final = response.choices[0] assert "logprobs" in choice assert "content" in choice.logprobs @@ -6933,11 +6953,9 @@ def test_completion_logprobs_stream_parses_top_logprobs(): "stream_options": {"include_usage": True}, "stream": True, } - assert [item.token for item in response_chunks[0].choices[0].logprobs.content[0].top_logprobs] == [ - "hello", - "hi", - "hey", - ] + assert [ + item.token for item in response_chunks[0].choices[0].logprobs.content[0].top_logprobs + ] == ["hello", "hi", "hey"] @respx.mock @@ -7008,7 +7026,9 @@ async def test_bedrock_converse_parses_scripted_response( "api_key": "test-bedrock-token", "aws_region_name": "us-west-2", } - response: Final = await litellm.acompletion(**request) if async_mode else litellm.completion(**request) + response: Final = ( + await litellm.acompletion(**request) if async_mode else litellm.completion(**request) + ) assert route.calls[0].request.headers["authorization"] == "Bearer test-bedrock-token" assert json.loads(route.calls[0].request.content) == { @@ -7095,7 +7115,9 @@ def test_bedrock_converse_parses_scripted_tool_call_arguments( tool_call: Final = response.choices[0].message.tool_calls[0] assert route.call_count == 1 assert tool_call.function.name == "trade" - assert json.loads(tool_call.function.arguments) == {"orders": [{"action": "buy", "asset": "BTC", "amount": 0.1}]} + assert json.loads(tool_call.function.arguments) == { + "orders": [{"action": "buy", "asset": "BTC", "amount": 0.1}] + } @pytest.mark.parametrize( @@ -7133,9 +7155,9 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( expected_request_body: dict[str, object], fake_provider_credentials: None, ): - route: Final = respx.post(f"https://bedrock-runtime.us-west-2.amazonaws.com/model/{invoke_model_id}/invoke").mock( - return_value=httpx.Response(200, json=response_body) - ) + route: Final = respx.post( + f"https://bedrock-runtime.us-west-2.amazonaws.com/model/{invoke_model_id}/invoke" + ).mock(return_value=httpx.Response(200, json=response_body)) response: Final = litellm.completion( model=model, @@ -7146,7 +7168,9 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( ) request: Final = route.calls[0].request - assert request.headers["authorization"].startswith("AWS4-HMAC-SHA256 Credential=unit-test/") + assert request.headers["authorization"].startswith( + "AWS4-HMAC-SHA256 Credential=unit-test/" + ) assert "/us-west-2/bedrock/aws4_request" in request.headers["authorization"] assert request.headers["x-amz-date"].endswith("Z") assert json.loads(request.content) == expected_request_body @@ -7156,9 +7180,9 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( @respx.mock def test_completion_azure_ad_token_is_forwarded(openai_api_response): - route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( - return_value=httpx.Response(200, json=openai_api_response) - ) + route: Final = respx.post( + "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" + ).mock(return_value=httpx.Response(200, json=openai_api_response)) litellm.completion( model="azure/gpt-6.1-sol", @@ -7173,9 +7197,9 @@ def test_completion_azure_ad_token_is_forwarded(openai_api_response): @respx.mock def test_completion_azure_extra_headers_are_forwarded(openai_api_response): - route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( - return_value=httpx.Response(200, json=openai_api_response) - ) + route: Final = respx.post( + "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" + ).mock(return_value=httpx.Response(200, json=openai_api_response)) litellm.completion( model="azure/gpt-6.1-sol", @@ -7196,9 +7220,9 @@ def test_completion_azure_extra_headers_are_forwarded(openai_api_response): @respx.mock def test_completion_azure_api_key_argument_is_forwarded(openai_api_response): - route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( - return_value=httpx.Response(200, json=openai_api_response) - ) + route: Final = respx.post( + "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" + ).mock(return_value=httpx.Response(200, json=openai_api_response)) response: Final = litellm.completion( model="azure/gpt-6.1-sol", @@ -7855,7 +7879,9 @@ def test_completion_deep_infra(drop_params): ], "usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25}, } - route: Final = respx.post(endpoint).mock(return_value=httpx.Response(200, json=response_body)) + route: Final = respx.post(endpoint).mock( + return_value=httpx.Response(200, json=response_body) + ) if not drop_params: with pytest.raises(litellm.exceptions.UnsupportedParamsError): @@ -7919,7 +7945,9 @@ def test_completion_deep_infra_mistral(): ], "usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25}, } - route: Final = respx.post(endpoint).mock(return_value=httpx.Response(200, json=response_body)) + route: Final = respx.post(endpoint).mock( + return_value=httpx.Response(200, json=response_body) + ) response: Final = completion( model=model_name, @@ -7968,13 +7996,19 @@ def test_unified_auth_params(provider, model, project, region_name, token): translated_optional_params = litellm.utils.get_optional_params(**data) if provider == "azure": - special_auth_params = litellm.AzureOpenAIConfig().get_mapped_special_auth_params() + special_auth_params = ( + litellm.AzureOpenAIConfig().get_mapped_special_auth_params() + ) elif provider == "bedrock": - special_auth_params = litellm.AmazonBedrockGlobalConfig().get_mapped_special_auth_params() + special_auth_params = ( + litellm.AmazonBedrockGlobalConfig().get_mapped_special_auth_params() + ) elif provider == "vertex_ai": special_auth_params = litellm.VertexAIConfig().get_mapped_special_auth_params() elif provider == "watsonx": - special_auth_params = litellm.IBMWatsonXAIConfig().get_mapped_special_auth_params() + special_auth_params = ( + litellm.IBMWatsonXAIConfig().get_mapped_special_auth_params() + ) for param, value in special_auth_params.items(): assert param in data @@ -8006,7 +8040,9 @@ async def test_dynamic_azure_params(stream, sync_mode): mock_client = AsyncMock(return_value="Hello world!") ## CHECK IF CLIENT IS USED (NO PARAM CHANGE) - with patch.object(client.chat.completions.with_raw_response, "create", new=mock_client) as mock_client: + with patch.object( + client.chat.completions.with_raw_response, "create", new=mock_client + ) as mock_client: try: # client.chat.completions.with_raw_response.create = mock_client if sync_mode: @@ -8035,7 +8071,9 @@ async def test_dynamic_azure_params(stream, sync_mode): new_mock_client = AsyncMock(return_value="Hello world!") ## CHECK IF NEW CLIENT IS USED (PARAM CHANGE) - with patch.object(client.chat.completions.with_raw_response, "create", new=new_mock_client) as new_mock_client: + with patch.object( + client.chat.completions.with_raw_response, "create", new=new_mock_client + ) as new_mock_client: try: if sync_mode: _ = completion( @@ -8246,7 +8284,9 @@ def test_completion_novita_ai(): openai_client = OpenAI(api_key="fake-key") - with patch.object(openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse)) as mock_call: + with patch.object( + openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse) + ) as mock_call: mock_call.return_value.headers = {} mock_call.return_value.parse.return_value = litellm.ModelResponse( choices=[{"message": {"role": "assistant", "content": "Hello"}}] @@ -8263,7 +8303,10 @@ def test_completion_novita_ai(): assert response.choices[0].message.content == "Hello" # Verify model is passed correctly - assert mock_call.call_args.kwargs["body"]["model"] == "meta-llama/llama-3.3-70b-instruct" + assert ( + mock_call.call_args.kwargs["body"]["model"] + == "meta-llama/llama-3.3-70b-instruct" + ) # Verify messages are passed correctly assert mock_call.call_args.kwargs["body"]["messages"] == messages @@ -8320,7 +8363,9 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features): litellm.set_verbose = True monkeypatch.setattr(litellm, "enable_preview_features", enable_preview_features) - with patch.object(client, "post", return_value=MagicMock()) as mock_completion: + with patch.object( + client, "post", return_value=MagicMock() + ) as mock_completion: try: resp = litellm.completion( model="openai/gpt-3.5-turbo", @@ -8333,7 +8378,9 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features): mock_completion.assert_called_once() if enable_preview_features: - assert mock_completion.call_args.kwargs["body"]["metadata"] == {"my-test-key": "my-test-value"} + assert mock_completion.call_args.kwargs["body"]["metadata"] == { + "my-test-key": "my-test-value" + } else: assert "metadata" not in mock_completion.call_args.kwargs["body"] @@ -8554,30 +8601,17 @@ async def test_async_responses_still_retries_provider_server_errors(monkeypatch) monkeypatch.setattr(litellm, "num_retries", None) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) with respx.mock as upstream: - response = upstream.post("https://openai-test.invalid/v1/responses").mock( - side_effect=[ - httpx.Response(500, json={"error": {"message": "temporary provider failure", "type": "server_error"}}), - httpx.Response( - 200, - json={ - "id": "resp-retry", - "object": "response", - "created_at": 1, - "status": "completed", - "model": "test-model", - "output": [], - "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, - }, - ), - ] - ) + response = upstream.post("https://openai-test.invalid/v1/responses").mock(side_effect=[ + httpx.Response(500, json={"error": {"message": "temporary provider failure", "type": "server_error"}}), + httpx.Response(200, json={ + "id": "resp-retry", "object": "response", "created_at": 1, "status": "completed", + "model": "test-model", "output": [], + "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, + }), + ]) result = await litellm.aresponses( - model="openai/test-model", - input="ping", - api_key="test-key", - api_base="https://openai-test.invalid/v1", - num_retries=1, - max_retries=0, + model="openai/test-model", input="ping", api_key="test-key", + api_base="https://openai-test.invalid/v1", num_retries=1, max_retries=0, ) assert result.status == "completed" assert response.call_count == 2 @@ -8763,7 +8797,9 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini", - "choices": [{"index": 0, "delta": {}, "finish_reason": "function_call"}], + "choices": [ + {"index": 0, "delta": {}, "finish_reason": "function_call"} + ], }, ) @@ -8787,7 +8823,9 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( ), ) ) - route: Final = respx.post(f"{api_base}/chat/completions").mock(side_effect=lambda _: next(scripted_responses)) + route: Final = respx.post(f"{api_base}/chat/completions").mock( + side_effect=lambda _: next(scripted_responses) + ) stream: Final = litellm.completion( model="gpt-4o-mini", api_key="sk-test", @@ -8798,14 +8836,11 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( ) answer_chunks: Final = tuple(stream) - assert ( - "".join( - chunk.choices[0].delta.content - for chunk in answer_chunks - if chunk.choices and chunk.choices[0].delta.content - ) - == "The answer is 42." - ) + assert "".join( + chunk.choices[0].delta.content + for chunk in answer_chunks + if chunk.choices and chunk.choices[0].delta.content + ) == "The answer is 42." assert tuple( chunk.choices[0].finish_reason for chunk in answer_chunks @@ -8822,7 +8857,12 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( assert answer_chunks[-1].usage.completion_tokens == 2 assert answer_chunks[-1].usage.total_tokens == 6 assert stream._hidden_params["api_base"] == api_base - assert answer_chunks[0]._hidden_params["additional_headers"]["llm_provider-x-ratelimit-remaining-requests"] == "17" + assert ( + answer_chunks[0]._hidden_params["additional_headers"][ + "llm_provider-x-ratelimit-remaining-requests" + ] + == "17" + ) function_stream: Final = litellm.completion( model="gpt-4o-mini", @@ -8834,7 +8874,9 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( stream=True, ) function_chunks: Final = tuple(function_stream) - assembled: Final = litellm.stream_chunk_builder(chunks=function_chunks, messages=messages) + assembled: Final = litellm.stream_chunk_builder( + chunks=function_chunks, messages=messages + ) function_call: Final = assembled.choices[0].message.function_call assert function_call is not None @@ -8846,8 +8888,12 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( if chunk.choices and chunk.choices[0].finish_reason is not None ) == ("function_call",) assert route.call_count == 2 - assert json.loads(route.calls[0].request.content)["stream_options"] == {"include_usage": True} - assert json.loads(route.calls[1].request.content)["function_call"] == {"name": "get_current_weather"} + assert json.loads(route.calls[0].request.content)["stream_options"] == { + "include_usage": True + } + assert json.loads(route.calls[1].request.content)["function_call"] == { + "name": "get_current_weather" + } assert json.loads(route.calls[1].request.content)["functions"] == list(functions) @@ -8886,9 +8932,15 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, }, ) - body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" - route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( - return_value=httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) + body: Final = "".join( + f"data: {json.dumps(event)}\n\n" for event in events + ) + "data: [DONE]\n\n" + route: Final = respx.post( + "https://api.openai.com/v1/chat/completions" + ).mock( + return_value=httpx.Response( + 200, text=body, headers={"content-type": "text/event-stream"} + ) ) stream: Final = await litellm.acompletion( model="gpt-4o-mini", @@ -8899,10 +8951,11 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( ) chunks: Final = tuple([chunk async for chunk in stream]) - assert ( - "".join(chunk.choices[0].delta.content for chunk in chunks if chunk.choices and chunk.choices[0].delta.content) - == "async reply" - ) + assert "".join( + chunk.choices[0].delta.content + for chunk in chunks + if chunk.choices and chunk.choices[0].delta.content + ) == "async reply" assert tuple(getattr(chunk, "usage", None) for chunk in chunks[:-1]) == ( None, None, @@ -8910,7 +8963,9 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( assert chunks[-1].usage.prompt_tokens == 3 assert chunks[-1].usage.completion_tokens == 2 assert chunks[-1].usage.total_tokens == 5 - assert json.loads(route.calls[0].request.content)["stream_options"] == {"include_usage": True} + assert json.loads(route.calls[0].request.content)["stream_options"] == { + "include_usage": True + } @respx.mock @@ -8947,10 +9002,16 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], }, ) - body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" - return httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) + body: Final = "".join( + f"data: {json.dumps(event)}\n\n" for event in events + ) + "data: [DONE]\n\n" + return httpx.Response( + 200, text=body, headers={"content-type": "text/event-stream"} + ) - route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(side_effect=respond) + route: Final = respx.post( + "https://api.openai.com/v1/chat/completions" + ).mock(side_effect=respond) async def collect(prompt: str) -> tuple[str, tuple[str, ...]]: stream = await litellm.acompletion( @@ -8960,7 +9021,9 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( stream=True, ) chunks: Final = tuple([chunk async for chunk in stream]) - response_text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) + response_text: Final = "".join( + chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices + ) finish_reasons: Final = tuple( chunk.choices[0].finish_reason for chunk in chunks @@ -8968,7 +9031,9 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( ) return response_text, finish_reasons - responses: Final = tuple(await asyncio.gather(collect("request one"), collect("request two"))) + responses: Final = tuple( + await asyncio.gather(collect("request one"), collect("request two")) + ) assert tuple(response for response, _ in responses) == ( "response one", @@ -9162,7 +9227,9 @@ def test_streaming_openai_usage_includes_test_owned_cost( "object": "chat.completion.chunk", "created": 1, "model": model_name, - "choices": [{"index": 0, "delta": {"role": "assistant", "content": "priced"}, "finish_reason": None}], + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": "priced"}, "finish_reason": None} + ], }, { "id": "chatcmpl-cost", @@ -9197,7 +9264,9 @@ def test_streaming_openai_usage_includes_test_owned_cost( ) stream_chunks: Final = tuple(stream) usage_chunks: Final = tuple( - getattr(chunk, "usage", None) for chunk in stream_chunks if getattr(chunk, "usage", None) is not None + getattr(chunk, "usage", None) + for chunk in stream_chunks + if getattr(chunk, "usage", None) is not None ) assert len(usage_chunks) == 1 assert usage_chunks[0].cost == pytest.approx(4 * 0.001 + 2 * 0.002) @@ -9213,7 +9282,9 @@ def test_completion_ensure_alternating_roles_sends_continue_messages(respx_mock: "object": "chat.completion", "created": 0, "model": "databricks-meta-llama-3-1-70b-instruct", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "Sure"}, "finish_reason": "stop"}], + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Sure"}, "finish_reason": "stop"} + ], "usage": {"prompt_tokens": 40, "completion_tokens": 1, "total_tokens": 41}, }, ) @@ -9423,12 +9494,9 @@ def test_azure_ai_forwards_reasoning_effort_once_the_deployment_declares_levels( assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "adaptive", _KIMI_DEPLOYMENT)["reasoning_effort"] == ( "adaptive" ) - assert ( - _chat_body_sent(respx_mock, "azure_ai/my-kimi", "none", {"reasoning_effort_levels": ["low", "high"]})[ - "reasoning_effort" - ] - == "none" - ) + assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "none", {"reasoning_effort_levels": ["low", "high"]})[ + "reasoning_effort" + ] == "none" def test_a_model_map_declaration_alone_leaves_reasoning_effort_to_drop_params( @@ -9563,9 +9631,7 @@ def test_a_default_effort_beside_a_thinking_budget_sends_no_effort_on_a_declarin thinking={"type": "enabled", "budget_tokens": 2048, **display}, **default_effort, ) - adaptive_alone: Final = body_sent( - respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking={"type": "adaptive", **display} - ) + adaptive_alone: Final = body_sent(respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking={"type": "adaptive", **display}) assert "output_config" not in defaulted assert defaulted == adaptive_alone @@ -10070,9 +10136,7 @@ async def test_aspeech_answers_a_non_string_provider_hint_with_a_bad_request( hint: str, respx_mock: respx.MockRouter ) -> None: with pytest.raises(litellm.BadRequestError) as raised: - await litellm.aspeech( - model="kokoro", input="hello", voice="alloy", api_key="fake_openai_api_key", **{hint: 123} - ) + await litellm.aspeech(model="kokoro", input="hello", voice="alloy", api_key="fake_openai_api_key", **{hint: 123}) assert raised.value.status_code == 400 assert f"Invalid {hint}" in raised.value.message @@ -10262,9 +10326,7 @@ async def test_converse_fallback_prices_the_current_attempt( } if asynchronous: with pytest.raises(litellm.BadRequestError): - await litellm.acompletion( - model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs - ) + await litellm.acompletion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs) else: with pytest.raises(litellm.BadRequestError): litellm.completion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs) @@ -10288,7 +10350,9 @@ async def test_converse_fallback_prices_the_current_attempt( assert result is not None prices: Final = litellm.get_model_info(profile or model, custom_llm_provider="bedrock") expected: Final = ( - 0.001 + 0.002 if custom_price else prices["input_cost_per_token"] + prices["output_cost_per_token"] + 0.001 + 0.002 + if custom_price + else prices["input_cost_per_token"] + prices["output_cost_per_token"] ) active_logging: Final = response.logging_obj if stream else logging_obj assert active_logging.response_cost_calculator(result) == pytest.approx(expected) @@ -10403,9 +10467,7 @@ async def test_bedrock_strict_tools_limit_rejection_is_sent_again_without_strict async def test_bedrock_strict_tools_limit_rejection_reaches_the_caller_without_drop_params( asynchronous: bool, local_model_cost_map: None ) -> None: - sent, error = await _converse_with_a_strict_tools_limit_rejection_first( - asynchronous, stream="off", drop_params=None - ) + sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream="off", drop_params=None) assert error is not None and "Too many strict tools" in str(error) assert len(sent) == 1 diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 9bc3c64de08..4e8a206bc5f 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -933,11 +933,21 @@ _COST_MAP_ROW_SCHEMA: Final = { "cache_creation_input_token_cost": {"type": "number"}, "cache_creation_input_token_cost_above_1hr": {"type": "number"}, "cache_creation_input_token_cost_above_12hr": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": { + "type": "number" + }, "cache_creation_input_token_cost_above_12hr_flex": {"type": "number"}, "cache_creation_input_token_cost_above_12hr_priority": {"type": "number"}, "cache_creation_input_token_cost_above_12hr_batches": {"type": "number"}, @@ -7409,7 +7419,9 @@ async def test_router_resolves_an_oidc_api_key_only_on_a_proxy_level_deployment( try: with respx.mock(assert_all_called=True) as upstream: upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) - await router.acompletion(model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"}) + await router.acompletion( + model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"} + ) sent: Final = upstream.calls.last.request.headers["Authorization"] finally: router.discard() @@ -7477,9 +7489,7 @@ async def test_an_inline_oidc_api_key_reaches_the_endpoint_its_deployment_config with respx.mock(assert_all_called=False) as upstream: upstream.route().respond(200, json=_BEDROCK_REPLY) await router.acompletion( - model="claude", - messages=_PING, - aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com", + model="claude", messages=_PING, aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com" ) sent: Final = tuple( (call.request.url.host, call.request.headers["Authorization"]) for call in upstream.calls @@ -7688,9 +7698,7 @@ async def test_a_team_row_naming_an_oidc_credential_reaches_only_the_credentials ( lambda reference: LiteLLM_Params( **litellm.Router( - model_list=[ - {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}} - ] + model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}}] ).get_model_list(model_name="openai/*")[0]["litellm_params"] ), lambda reference: "Bearer listing-token", @@ -8239,17 +8247,10 @@ def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice): assert exc_info.value.model == MODEL -_ALLOWED_TOOLS_CHAT_LAYOUT: Final = { - "type": "allowed_tools", - "allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, -} - - @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("mode", ["auto", "required"]) -def test_validate_tool_choice_allowed_tools_chat_layout_passes_through(mode): - """OpenAI's chat allowed_tools restriction is forwarded as the caller sent it, not rejected as an unknown dict.""" - tool_choice = { +def test_validate_tool_choice_allowed_tools_chat_layout_passes_through(mode: str): + tool_choice: Final = { "type": "allowed_tools", "allowed_tools": { "mode": mode, @@ -8272,9 +8273,7 @@ def test_validate_tool_choice_allowed_tools_chat_layout_passes_through(mode): {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": "get_time"}}, ], ) -def test_validate_tool_choice_allowed_tools_malformed_is_a_400(tool_choice): - """The flat Responses layout, a missing or empty tool list, or a mode OpenAI does not define is a 400 that shows - the chat layout, since OpenAI's chat endpoint rejects each of those too.""" +def test_validate_tool_choice_allowed_tools_malformed_is_a_400(tool_choice: dict[str, object]): with pytest.raises(litellm.BadRequestError, match="allowed_tools") as exc_info: validate_chat_completion_tool_choice(tool_choice, model=MODEL) assert exc_info.value.status_code == 400 @@ -8705,9 +8704,7 @@ def test_basic_trimming_no_max_tokens_specified(): trimmed_messages = trim_messages(messages, model="gpt-4") print("trimmed messages for gpt-4") print(trimmed_messages) - assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"][ - "max_input_tokens" - ] + assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_input_tokens"] def test_multiple_messages_trimming():