From 348af8ab75825fb4fefe1a60f8c66904aaeb93d4 Mon Sep 17 00:00:00 2001 From: longwind48 Date: Sun, 23 Aug 2026 10:45:43 +0800 Subject: [PATCH 1/9] feat(bedrock_mantle): forward Bedrock's native web_search tool on Responses Mantle's Responses tool filter allowlisted function, mcp, custom, namespace and tool_search, so a `{"type": "web_search"}` tool was stripped before the request left LiteLLM. Bedrock has served Web Search server-side since 4 Aug 2026 and there was no way to reach it through the proxy. The tool is enabled per model, not per provider: measured against bedrock-mantle.us-east-1.api.aws, openai.gpt-5.6-sol/terra/luna, gpt-5.5 and gpt-5.4 all return web_search_call items plus url_citation annotations, while google.gemma-4-31b answers "Tool type 'web_search' is not supported for model `google.gemma-4-31b`". A flat allowlist entry would therefore turn today's silent drop into a hard 400 for the rest of the provider, so the filter reads the model's supports_web_search flag and the five capable ids now carry it in the cost map. Requests without a web_search tool never do the lookup. Response handling needs nothing new: Mantle emits the OpenAI Responses shapes LiteLLM already models, so url_citation annotations and web_search_call items pass through untouched. --- .../responses/transformation.py | 54 ++++++--- ...odel_prices_and_context_window_backup.json | 15 ++- model_prices_and_context_window.json | 15 ++- ...bedrock_mantle_responses_transformation.py | 104 +++++++++++++++++- 4 files changed, 157 insertions(+), 31 deletions(-) diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index 92da5835b2d..012688e2e5f 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -15,6 +15,7 @@ role / access key / profile / web identity), signed via the shared BaseAWSLLM._sign_request after the request body is finalized. """ +from collections.abc import Sequence from typing import Any, Final import litellm @@ -44,7 +45,18 @@ _BASE_SUFFIXES_TO_STRIP: Final = ( ) # Per Bedrock Mantle Responses API validation errors. -_BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES = frozenset({"function", "mcp", "custom", "namespace", "tool_search"}) +_BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES: Final = frozenset( + {"function", "mcp", "custom", "namespace", "tool_search"} +) + +# Bedrock's server-side Web Search tool is enabled per model, not per provider: the GPT +# families AWS lists accept it, the rest reject the whole request with "Tool type +# 'web_search' is not supported for model ``", so the cost map flag decides. +_BEDROCK_MANTLE_WEB_SEARCH_TOOL_TYPE: Final = "web_search" + +_BEDROCK_MANTLE_RESPONSE_TOOL_TYPES_WITH_WEB_SEARCH: Final = _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES | frozenset( + (_BEDROCK_MANTLE_WEB_SEARCH_TOOL_TYPE,) +) _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"}) @@ -100,9 +112,20 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI def supports_native_websocket(self) -> bool: return False - @staticmethod - def _filter_unsupported_tools(tools: list[Any]) -> list[Any]: - """Keep only tool types Mantle's Responses API accepts.""" + def _supported_response_tool_types(self, tools: "Sequence[Any]", model: str) -> frozenset[str]: + """Only consult the cost map when the request actually carries a Web Search tool, so + every other request keeps resolving its tool types without a model lookup.""" + if not any( + isinstance(tool, dict) and tool.get("type") == _BEDROCK_MANTLE_WEB_SEARCH_TOOL_TYPE for tool in tools + ): + return _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES + if litellm.supports_web_search(model=model, custom_llm_provider=self.custom_llm_provider.value): + return _BEDROCK_MANTLE_RESPONSE_TOOL_TYPES_WITH_WEB_SEARCH + return _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES + + def _filter_unsupported_tools(self, tools: list[Any], model: str) -> list[Any]: + """Keep only tool types Mantle's Responses API accepts for this model.""" + supported_tool_types: Final = self._supported_response_tool_types(tools=tools, model=model) kept: Final[list[Any]] = [] dropped_types: Final[list[str]] = [] for tool in tools: @@ -110,16 +133,17 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI kept.append(tool) continue tool_type = tool.get("type") - if tool_type in _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES: + if tool_type in supported_tool_types: kept.append(tool) else: dropped_types.append(str(tool_type)) if dropped_types: verbose_logger.warning( - "Bedrock Mantle Responses API: dropping unsupported tool type(s) %s (supported: %s).", + "Bedrock Mantle Responses API: dropping unsupported tool type(s) %s for model %s (supported: %s).", sorted(set(dropped_types)), - sorted(_BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES), + model, + sorted(supported_tool_types), ) return kept @@ -154,7 +178,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI litellm_params: GenericLiteLLMParams, headers: dict, ) -> dict: - remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input) + remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input=input, model=model) request_params: Final = ( { **response_api_optional_request_params, @@ -183,10 +207,10 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI tools: Final = item.get("tools") return tools if isinstance(tools, list) else [] - @classmethod def _hoist_codex_additional_tools( - cls, + self, input: "str | ResponseInputParam", + model: str, ) -> "tuple[str | ResponseInputParam, list[Any]]": """Codex's "responses lite" wire mode ships tool definitions inside `input` as {"type": "additional_tools", "role": "developer", @@ -197,18 +221,18 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI """ if not isinstance(input, list): return input, [] - additional_tools_items: Final = [item for item in input if cls._is_codex_additional_tools_item(item)] + additional_tools_items: Final = [item for item in input if self._is_codex_additional_tools_item(item)] if not additional_tools_items: return input, [] - remaining_input: Final = [item for item in input if not cls._is_codex_additional_tools_item(item)] - hoisted_tools = [tool for item in additional_tools_items for tool in cls._tools_of_additional_tools_item(item)] + remaining_input: Final = [item for item in input if not self._is_codex_additional_tools_item(item)] + hoisted_tools = [tool for item in additional_tools_items for tool in self._tools_of_additional_tools_item(item)] verbose_logger.debug( "Bedrock Mantle Responses API: hoisting %d tool(s) out of %d 'additional_tools' input item(s) " "into the top-level tools param (Mantle rejects that input item type).", len(hoisted_tools), len(additional_tools_items), ) - return remaining_input, cls._filter_unsupported_tools(hoisted_tools) + return remaining_input, self._filter_unsupported_tools(tools=hoisted_tools, model=model) def map_openai_params( self, @@ -230,7 +254,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI return params tools_list: Final = tools if isinstance(tools, list) else [tools] - filtered: Final = self._filter_unsupported_tools(tools_list) + filtered: Final = self._filter_unsupported_tools(tools=tools_list, model=model) if filtered: params["tools"] = filtered else: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3af7d9e5019..f0424f674e2 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48615,7 +48615,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, @@ -48647,7 +48648,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, @@ -48679,7 +48681,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "us.openai.gpt-5.6-sol": { "input_cost_per_token": 5.5e-06, @@ -48856,7 +48859,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.4": { "input_cost_per_token": 2.75e-06, @@ -48883,7 +48887,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/google.gemma-4-31b": { "input_cost_per_token": 1.4e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3af7d9e5019..f0424f674e2 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48615,7 +48615,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, @@ -48647,7 +48648,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, @@ -48679,7 +48681,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "us.openai.gpt-5.6-sol": { "input_cost_per_token": 5.5e-06, @@ -48856,7 +48859,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.4": { "input_cost_per_token": 2.75e-06, @@ -48883,7 +48887,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/google.gemma-4-31b": { "input_cost_per_token": 1.4e-07, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 9e05d48a18f..97c4da654f3 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -330,7 +330,7 @@ class TestBedrockMantleResponsesTools: params = cfg.map_openai_params( response_api_optional_params={ "tools": [ - {"type": "web_search"}, + {"type": "file_search"}, {"type": "function", "name": "exec_command"}, ] }, @@ -342,7 +342,7 @@ class TestBedrockMantleResponsesTools: def test_map_openai_params_removes_tools_when_all_unsupported(self): cfg = BedrockMantleResponsesAPIConfig() params = cfg.map_openai_params( - response_api_optional_params={"tools": [{"type": "web_search"}]}, + response_api_optional_params={"tools": [{"type": "file_search"}]}, model="openai.gpt-5.5", drop_params=False, ) @@ -356,12 +356,104 @@ class TestBedrockMantleResponsesTools: "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning" ) as mock_warning: cfg.map_openai_params( - response_api_optional_params={"tools": [{"type": "web_search"}]}, + response_api_optional_params={"tools": [{"type": "file_search"}]}, model="openai.gpt-5.5", drop_params=False, ) assert mock_warning.call_count == 1 - assert "web_search" in str(mock_warning.call_args) + assert "file_search" in str(mock_warning.call_args) + + +class TestBedrockMantleResponsesNativeWebSearch: + """Bedrock serves Web Search server-side on the bedrock-mantle Responses API for the + GPT families AWS enables (verified against bedrock-mantle.us-east-1.api.aws: sol, + terra, luna, gpt-5.5 and gpt-5.4 all return `web_search_call` items and + `url_citation` annotations). Other Mantle models reject the tool outright, so the + filter has to gate on the model's `supports_web_search` flag rather than forward it + for the whole provider.""" + + _WEB_SEARCH_TOOL = {"type": "web_search", "external_web_access": False} + _FUNCTION_TOOL = {"type": "function", "name": "exec_command"} + + @pytest.mark.parametrize( + "model", + [ + "openai.gpt-5.6-sol", + "openai.gpt-5.6-terra", + "openai.gpt-5.6-luna", + "openai.gpt-5.5", + "openai.gpt-5.4", + "bedrock_mantle/openai.gpt-5.6-terra", + ], + ) + def test_web_search_survives_for_capable_models(self, model, local_model_cost_map): + cfg = BedrockMantleResponsesAPIConfig() + params = cfg.map_openai_params( + response_api_optional_params={"tools": [self._WEB_SEARCH_TOOL, self._FUNCTION_TOOL]}, + model=model, + drop_params=False, + ) + assert params["tools"] == [self._WEB_SEARCH_TOOL, self._FUNCTION_TOOL] + + def test_external_web_access_true_is_forwarded_verbatim(self, local_model_cost_map): + cfg = BedrockMantleResponsesAPIConfig() + params = cfg.map_openai_params( + response_api_optional_params={"tools": [{"type": "web_search", "external_web_access": True}]}, + model="openai.gpt-5.6-terra", + drop_params=False, + ) + assert params["tools"] == [{"type": "web_search", "external_web_access": True}] + + @pytest.mark.parametrize("model", ["google.gemma-4-31b", "openai.gpt-oss-120b"]) + def test_web_search_still_dropped_for_models_without_the_capability(self, model, local_model_cost_map): + cfg = BedrockMantleResponsesAPIConfig() + params = cfg.map_openai_params( + response_api_optional_params={"tools": [self._WEB_SEARCH_TOOL, self._FUNCTION_TOOL]}, + model=model, + drop_params=False, + ) + assert params["tools"] == [self._FUNCTION_TOOL] + + def test_web_search_hoisted_out_of_codex_additional_tools(self, local_model_cost_map): + cfg = BedrockMantleResponsesAPIConfig() + body = cfg.transform_responses_api_request( + model="openai.gpt-5.6-terra", + input=[ + {"type": "additional_tools", "role": "developer", "tools": [self._WEB_SEARCH_TOOL]}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ], + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert body["tools"] == [self._WEB_SEARCH_TOOL] + + def test_web_search_hoisted_from_codex_is_dropped_for_incapable_model(self, local_model_cost_map): + cfg = BedrockMantleResponsesAPIConfig() + body = cfg.transform_responses_api_request( + model="google.gemma-4-31b", + input=[ + {"type": "additional_tools", "role": "developer", "tools": [self._WEB_SEARCH_TOOL]}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ], + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert "tools" not in body + + @pytest.mark.parametrize( + "model", + [ + "bedrock_mantle/openai.gpt-5.6-sol", + "bedrock_mantle/openai.gpt-5.6-terra", + "bedrock_mantle/openai.gpt-5.6-luna", + "bedrock_mantle/openai.gpt-5.5", + "bedrock_mantle/openai.gpt-5.4", + ], + ) + def test_cost_map_advertises_web_search(self, model, local_model_cost_map): + assert litellm.supports_web_search(model=model, custom_llm_provider="bedrock_mantle") is True def _codex_exec_tool(): @@ -549,7 +641,7 @@ class TestBedrockMantleCodexAdditionalTools: "type": "additional_tools", "role": "developer", "tools": [ - {"type": "web_search"}, + {"type": "file_search"}, {"type": "function", "name": "wait"}, ], }, @@ -561,7 +653,7 @@ class TestBedrockMantleCodexAdditionalTools: def test_item_stripped_even_when_no_hoisted_tool_survives(self): body = self._transform( input=[ - {"type": "additional_tools", "role": "developer", "tools": [{"type": "web_search"}]}, + {"type": "additional_tools", "role": "developer", "tools": [{"type": "file_search"}]}, self._USER_MESSAGE, ] ) From ce80105a4f4f015b998e441f16b1a4d0efc3b030 Mon Sep 17 00:00:00 2001 From: longwind48 Date: Sun, 23 Aug 2026 11:03:42 +0800 Subject: [PATCH 2/9] test(bedrock_mantle): close the web_search guard's mutation gap Deleting the early return that skips the capability lookup for requests carrying no web_search tool killed no test, so the guard was untested. Assert the advertised tool set widens only when the request asks for it. Also stop comparing the class-level tool dicts against themselves: the assertions passed even if the transform mutated a tool in place, and the shared dicts could leak across tests. Build them from factories, matching _codex_exec_tool below. Fold the five parametrized cost-map assertions into one, drop the comment that restated the module comment, and align _filter_unsupported_tools on Sequence[Any] with its new sibling. --- .../responses/transformation.py | 12 +- ...bedrock_mantle_responses_transformation.py | 210 +++++++----------- 2 files changed, 83 insertions(+), 139 deletions(-) diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index 012688e2e5f..819092517df 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -49,9 +49,9 @@ _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES: Final = frozenset( {"function", "mcp", "custom", "namespace", "tool_search"} ) -# Bedrock's server-side Web Search tool is enabled per model, not per provider: the GPT -# families AWS lists accept it, the rest reject the whole request with "Tool type -# 'web_search' is not supported for model ``", so the cost map flag decides. +# Enabled per model, not per provider: models AWS has not enabled reject the whole request +# with "Tool type 'web_search' is not supported for model ``", so the cost map's +# supports_web_search flag decides rather than a provider-wide allowlist entry. _BEDROCK_MANTLE_WEB_SEARCH_TOOL_TYPE: Final = "web_search" _BEDROCK_MANTLE_RESPONSE_TOOL_TYPES_WITH_WEB_SEARCH: Final = _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES | frozenset( @@ -113,8 +113,8 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI return False def _supported_response_tool_types(self, tools: "Sequence[Any]", model: str) -> frozenset[str]: - """Only consult the cost map when the request actually carries a Web Search tool, so - every other request keeps resolving its tool types without a model lookup.""" + """The tool types `tools` may keep: the provider-wide set, widened by `web_search` + only when this request asks for it and the model is one AWS enabled it for.""" if not any( isinstance(tool, dict) and tool.get("type") == _BEDROCK_MANTLE_WEB_SEARCH_TOOL_TYPE for tool in tools ): @@ -123,7 +123,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI return _BEDROCK_MANTLE_RESPONSE_TOOL_TYPES_WITH_WEB_SEARCH return _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES - def _filter_unsupported_tools(self, tools: list[Any], model: str) -> list[Any]: + def _filter_unsupported_tools(self, tools: "Sequence[Any]", model: str) -> list[Any]: """Keep only tool types Mantle's Responses API accepts for this model.""" supported_tool_types: Final = self._supported_response_tool_types(tools=tools, model=model) kept: Final[list[Any]] = [] diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 97c4da654f3..87177ae07fb 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -46,10 +46,7 @@ class TestBedrockMantleResponsesURL: api_base="https://bedrock-mantle.us-east-2.api.aws/v1/", litellm_params={}, ) - assert ( - url_trailing - == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" - ) + assert url_trailing == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" def test_url_does_not_double_openai_v1(self, monkeypatch): monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) @@ -109,9 +106,7 @@ class TestBedrockMantleResponsesURL: with pytest.raises(ValueError, match="api\\.aws\\.attacker\\.example/'\\. Region names must contain only"): cfg.get_complete_url( api_base=None, - litellm_params={ - "aws_region_name": "us-east-1.api.aws.attacker.example/" - }, + litellm_params={"aws_region_name": "us-east-1.api.aws.attacker.example/"}, ) def test_url_region_default_us_east_1(self, monkeypatch): @@ -151,7 +146,6 @@ class TestBedrockMantleResponsesURL: assert url == "https://bedrock-mantle.us-east-2.api.aws/v1/responses" assert url.count("/responses") == 1 - def test_url_aws_region_name_overrides_stale_api_base(self, monkeypatch): monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) @@ -165,9 +159,7 @@ class TestBedrockMantleResponsesURL: class TestBedrockMantleGetLlmProviderRegion: - def test_get_llm_provider_uses_supplemental_litellm_params( - self, monkeypatch, local_cost_map - ): + def test_get_llm_provider_uses_supplemental_litellm_params(self, monkeypatch, local_cost_map): monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) @@ -184,9 +176,7 @@ class TestBedrockMantleGetLlmProviderRegion: # the resolved chat base) is on the /openai/v1 base per the AWS card. assert api_base == "https://bedrock-mantle.us-east-2.api.aws/openai/v1" - def test_get_llm_provider_uses_aws_region_from_litellm_params( - self, monkeypatch, local_cost_map - ): + def test_get_llm_provider_uses_aws_region_from_litellm_params(self, monkeypatch, local_cost_map): monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) @@ -220,18 +210,14 @@ class TestBedrockMantleResponsesAuth: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-key") monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment( - headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() - ) + headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) assert headers["Authorization"] == "Bearer env-key" def test_bedrock_bearer_token_fallback(self, monkeypatch): monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "bearer-key") cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment( - headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() - ) + headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) assert headers["Authorization"] == "Bearer bearer-key" def test_missing_bearer_does_not_raise_in_validate_environment(self, monkeypatch): @@ -239,9 +225,7 @@ class TestBedrockMantleResponsesAuth: monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment( - headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() - ) + headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) assert "Authorization" not in headers def test_project_id_sets_openai_project_header(self): @@ -249,9 +233,7 @@ class TestBedrockMantleResponsesAuth: headers = cfg.validate_environment( headers={}, model="openai.gpt-5.5", - litellm_params=GenericLiteLLMParams( - api_key="fake-key", aws_bedrock_project_id="proj_abc123def456" - ), + litellm_params=GenericLiteLLMParams(api_key="fake-key", aws_bedrock_project_id="proj_abc123def456"), ) assert headers["OpenAI-Project"] == "proj_abc123def456" @@ -352,9 +334,7 @@ class TestBedrockMantleResponsesTools: from unittest.mock import patch cfg = BedrockMantleResponsesAPIConfig() - with patch( - "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning" - ) as mock_warning: + with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning") as mock_warning: cfg.map_openai_params( response_api_optional_params={"tools": [{"type": "file_search"}]}, model="openai.gpt-5.5", @@ -364,96 +344,100 @@ class TestBedrockMantleResponsesTools: assert "file_search" in str(mock_warning.call_args) +def _web_search_tool(external_web_access=False): + return {"type": "web_search", "external_web_access": external_web_access} + + +def _function_tool(): + return {"type": "function", "name": "exec_command"} + + +def _codex_additional_tools_input(tools): + return [ + {"type": "additional_tools", "role": "developer", "tools": tools}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ] + + +_WEB_SEARCH_CAPABLE_MODELS = [ + "openai.gpt-5.6-sol", + "openai.gpt-5.6-terra", + "openai.gpt-5.6-luna", + "openai.gpt-5.5", + "openai.gpt-5.4", +] + + class TestBedrockMantleResponsesNativeWebSearch: - """Bedrock serves Web Search server-side on the bedrock-mantle Responses API for the - GPT families AWS enables (verified against bedrock-mantle.us-east-1.api.aws: sol, - terra, luna, gpt-5.5 and gpt-5.4 all return `web_search_call` items and - `url_citation` annotations). Other Mantle models reject the tool outright, so the - filter has to gate on the model's `supports_web_search` flag rather than forward it - for the whole provider.""" + """Verified against bedrock-mantle.us-east-1.api.aws: every id in + _WEB_SEARCH_CAPABLE_MODELS returns `web_search_call` items and `url_citation` + annotations, while google.gemma-4-31b answers "Tool type 'web_search' is not + supported for model `google.gemma-4-31b`".""" - _WEB_SEARCH_TOOL = {"type": "web_search", "external_web_access": False} - _FUNCTION_TOOL = {"type": "function", "name": "exec_command"} - - @pytest.mark.parametrize( - "model", - [ - "openai.gpt-5.6-sol", - "openai.gpt-5.6-terra", - "openai.gpt-5.6-luna", - "openai.gpt-5.5", - "openai.gpt-5.4", - "bedrock_mantle/openai.gpt-5.6-terra", - ], - ) + @pytest.mark.parametrize("model", [*_WEB_SEARCH_CAPABLE_MODELS, "bedrock_mantle/openai.gpt-5.6-terra"]) def test_web_search_survives_for_capable_models(self, model, local_model_cost_map): cfg = BedrockMantleResponsesAPIConfig() params = cfg.map_openai_params( - response_api_optional_params={"tools": [self._WEB_SEARCH_TOOL, self._FUNCTION_TOOL]}, + response_api_optional_params={"tools": [_web_search_tool(), _function_tool()]}, model=model, drop_params=False, ) - assert params["tools"] == [self._WEB_SEARCH_TOOL, self._FUNCTION_TOOL] + assert params["tools"] == [_web_search_tool(), _function_tool()] def test_external_web_access_true_is_forwarded_verbatim(self, local_model_cost_map): cfg = BedrockMantleResponsesAPIConfig() params = cfg.map_openai_params( - response_api_optional_params={"tools": [{"type": "web_search", "external_web_access": True}]}, + response_api_optional_params={"tools": [_web_search_tool(external_web_access=True)]}, model="openai.gpt-5.6-terra", drop_params=False, ) - assert params["tools"] == [{"type": "web_search", "external_web_access": True}] + assert params["tools"] == [_web_search_tool(external_web_access=True)] @pytest.mark.parametrize("model", ["google.gemma-4-31b", "openai.gpt-oss-120b"]) def test_web_search_still_dropped_for_models_without_the_capability(self, model, local_model_cost_map): cfg = BedrockMantleResponsesAPIConfig() params = cfg.map_openai_params( - response_api_optional_params={"tools": [self._WEB_SEARCH_TOOL, self._FUNCTION_TOOL]}, + response_api_optional_params={"tools": [_web_search_tool(), _function_tool()]}, model=model, drop_params=False, ) - assert params["tools"] == [self._FUNCTION_TOOL] + assert params["tools"] == [_function_tool()] + + def test_web_search_is_not_advertised_unless_the_request_asks_for_it(self, local_model_cost_map): + """The tool type widens only for requests that carry it, so a capable model's + other requests keep the provider-wide set and never reach the cost map.""" + cfg = BedrockMantleResponsesAPIConfig() + supported = cfg._supported_response_tool_types(tools=[_function_tool()], model="openai.gpt-5.6-terra") + assert "web_search" not in supported + assert "function" in supported def test_web_search_hoisted_out_of_codex_additional_tools(self, local_model_cost_map): cfg = BedrockMantleResponsesAPIConfig() body = cfg.transform_responses_api_request( model="openai.gpt-5.6-terra", - input=[ - {"type": "additional_tools", "role": "developer", "tools": [self._WEB_SEARCH_TOOL]}, - {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, - ], + input=_codex_additional_tools_input([_web_search_tool()]), response_api_optional_request_params={}, litellm_params=GenericLiteLLMParams(), headers={}, ) - assert body["tools"] == [self._WEB_SEARCH_TOOL] + assert body["tools"] == [_web_search_tool()] def test_web_search_hoisted_from_codex_is_dropped_for_incapable_model(self, local_model_cost_map): cfg = BedrockMantleResponsesAPIConfig() body = cfg.transform_responses_api_request( model="google.gemma-4-31b", - input=[ - {"type": "additional_tools", "role": "developer", "tools": [self._WEB_SEARCH_TOOL]}, - {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, - ], + input=_codex_additional_tools_input([_web_search_tool()]), response_api_optional_request_params={}, litellm_params=GenericLiteLLMParams(), headers={}, ) assert "tools" not in body - @pytest.mark.parametrize( - "model", - [ - "bedrock_mantle/openai.gpt-5.6-sol", - "bedrock_mantle/openai.gpt-5.6-terra", - "bedrock_mantle/openai.gpt-5.6-luna", - "bedrock_mantle/openai.gpt-5.5", - "bedrock_mantle/openai.gpt-5.4", - ], - ) - def test_cost_map_advertises_web_search(self, model, local_model_cost_map): - assert litellm.supports_web_search(model=model, custom_llm_provider="bedrock_mantle") is True + def test_cost_map_advertises_web_search(self, local_model_cost_map): + assert all( + litellm.supports_web_search(model=f"bedrock_mantle/{model}", custom_llm_provider="bedrock_mantle") + for model in _WEB_SEARCH_CAPABLE_MODELS + ) def _codex_exec_tool(): @@ -531,9 +515,7 @@ class TestBedrockMantleServiceTier: from unittest.mock import patch cfg = BedrockMantleResponsesAPIConfig() - with patch( - "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning" - ) as mock_warning: + with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning") as mock_warning: cfg.map_openai_params( response_api_optional_params={"service_tier": "priority"}, model="openai.gpt-5.5", @@ -702,9 +684,7 @@ class TestBedrockMantleCodexAdditionalTools: def test_hoist_is_logged_at_debug_level(self): from unittest.mock import patch - with patch( - "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.debug" - ) as mock_debug: + with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.debug") as mock_debug: self._transform( input=[ {"type": "additional_tools", "role": "developer", "tools": self._CODEX_TOOLS}, @@ -839,9 +819,7 @@ class TestBedrockMantleResponsesRegistry: ) assert cfg is None - def test_price_map_flag_routes_non_gpt_name_to_openai_path( - self, restore_model_cost - ): + def test_price_map_flag_routes_non_gpt_name_to_openai_path(self, restore_model_cost): # Data-driven onboarding: a frontier model whose name does NOT match the # openai.gpt- convention can still be routed to /openai/v1/responses by # declaring use_openai_responses_path in its price-map entry, with no code @@ -867,18 +845,8 @@ class TestBedrockMantleResponsesRegistry: def test_gpt_5_5_price_map_declares_openai_responses_path(self, local_cost_map): # The gpt-5.x entries must carry the data-driven flag so frontier routing # does not rely on the name-string fallback alone. - assert ( - litellm.model_cost["bedrock_mantle/openai.gpt-5.5"].get( - "use_openai_responses_path" - ) - is True - ) - assert ( - litellm.model_cost["bedrock_mantle/openai.gpt-5.4"].get( - "use_openai_responses_path" - ) - is True - ) + assert litellm.model_cost["bedrock_mantle/openai.gpt-5.5"].get("use_openai_responses_path") is True + assert litellm.model_cost["bedrock_mantle/openai.gpt-5.4"].get("use_openai_responses_path") is True @pytest.mark.parametrize( "model", @@ -913,9 +881,7 @@ class TestBedrockMantleResponsesRegistry: ) assert cfg is None - def test_declared_responses_non_openai_routes_to_standard_path( - self, restore_model_cost - ): + def test_declared_responses_non_openai_routes_to_standard_path(self, restore_model_cost): # New feature: a non-OpenAI model declared mode=responses (e.g. via a # user's proxy model_info block) must route to the STANDARD /v1/responses # path, not the frontier /openai/v1/responses path. Fails before the @@ -1023,11 +989,7 @@ class TestMantleBaseSegment: ), ( "google.gemma-4-31b", - { - "bedrock_mantle/google.gemma-4-31b": { - "use_openai_responses_path": True - } - }, + {"bedrock_mantle/google.gemma-4-31b": {"use_openai_responses_path": True}}, "openai/v1", ), ( @@ -1066,11 +1028,7 @@ class TestMantleSupportsResponses: # chat-only supported_endpoints -> not supported (the discriminator) ( "openai.gpt-oss-safeguard-120b", - { - "bedrock_mantle/openai.gpt-oss-safeguard-120b": { - "supported_endpoints": ["/v1/chat/completions"] - } - }, + {"bedrock_mantle/openai.gpt-oss-safeguard-120b": {"supported_endpoints": ["/v1/chat/completions"]}}, False, ), # mode=responses (no supported_endpoints) -> supported @@ -1109,9 +1067,7 @@ class TestBedrockMantlePerModelResponsesURL: model=model, ) assert isinstance(cfg, BedrockMantleResponsesAPIConfig) - return cfg.get_complete_url( - api_base=None, litellm_params={"aws_region_name": region} - ) + return cfg.get_complete_url(api_base=None, litellm_params={"aws_region_name": region}) def test_gpt_oss_uses_standard_responses_path(self, local_cost_map): url = self._url_for("openai.gpt-oss-120b") @@ -1210,9 +1166,7 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) signer = BaseAWSLLM() - signer.get_credentials = MagicMock( - side_effect=AssertionError("get_credentials must not run for bearer auth") - ) + signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, signed_body = cfg.sign_request( @@ -1234,9 +1188,7 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-bearer") signer = BaseAWSLLM() - signer.get_credentials = MagicMock( - side_effect=AssertionError("get_credentials must not run for bearer auth") - ) + signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, _ = cfg.sign_request( @@ -1258,9 +1210,7 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-bearer") signer = BaseAWSLLM() - signer.get_credentials = MagicMock( - side_effect=AssertionError("get_credentials must not run for bearer auth") - ) + signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, _ = cfg.sign_request( @@ -1406,9 +1356,7 @@ class TestBedrockMantleResponsesSigV4: } cfg = BedrockMantleResponsesAPIConfig(aws_signer=BaseAWSLLM()) url = cfg.get_complete_url(api_base=None, litellm_params=params) - assert ( - url == "https://bedrock-mantle.ap-southeast-2.api.aws/openai/v1/responses" - ) + assert url == "https://bedrock-mantle.ap-southeast-2.api.aws/openai/v1/responses" headers, _ = cfg.sign_request( headers={}, @@ -1419,9 +1367,7 @@ class TestBedrockMantleResponsesSigV4: ) assert "/ap-southeast-2/bedrock/aws4_request" in headers["Authorization"] - def test_injected_default_region_base_does_not_override_aws_region_name( - self, monkeypatch - ): + def test_injected_default_region_base_does_not_override_aws_region_name(self, monkeypatch): """2nd-round adversarial regression: responses/main.py auto-injects litellm_params.api_base = https://bedrock-mantle..api.aws/v1 (default region, ignoring aws_region_name). The config must still pin BOTH the URL host @@ -1507,7 +1453,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=NoCredentialsError()) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: + with pytest.raises(ValueError, match="Bedrock Mantle auth failed: no Bearer token and no usable") as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -1537,7 +1483,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=cred_error) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: + with pytest.raises(ValueError, match="Bedrock Mantle auth failed: no Bearer token and no usable") as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -1562,9 +1508,7 @@ class TestBedrockMantleResponsesSigV4: signer = BaseAWSLLM() signer.get_credentials = MagicMock( - side_effect=ConnectTimeoutError( - endpoint_url="https://sts.us-east-2.amazonaws.com" - ) + side_effect=ConnectTimeoutError(endpoint_url="https://sts.us-east-2.amazonaws.com") ) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) From 7da19d26d6c195e1f220d334a32cdca44c710c02 Mon Sep 17 00:00:00 2001 From: longwind48 Date: Sun, 23 Aug 2026 16:42:06 +0800 Subject: [PATCH 3/9] fix(bedrock_mantle): bill Bedrock's native web searches instead of $0 The five ids that carry supports_web_search had no search_context_cost_per_query, so get_cost_for_web_search fell through to get_default_cost_for_web_search, which returns 0.0 when the table is absent. Every native Bedrock search billed nothing. AWS publishes the rate on the AmazonBedrock Bedrock-Websearch-Queries usage type at $12.00 per 1,000 queries, identical in all three Regions that offer Web Search (us-east-1 HTPA2KVTTT65JDW6, us-east-2 YV3KSHNXJCCU32NJ, us-west-2 E6H6Z5RYDRVX7HF9, effective 2026-08-01). That is a distinct meter from Web Search on AgentCore, which bills $7.00 per 1,000 on AmazonBedrockAgentCore, so the AgentCore rate does not apply here. The rate does not vary by search context size, so all three size keys carry it. Bedrock charges per search rather than per request, and _count_web_search_calls already multiplies the per-call rate by the number of web_search_call items, so a response that searched seven times now bills seven queries. --- ...odel_prices_and_context_window_backup.json | 25 +++++++++++++ model_prices_and_context_window.json | 25 +++++++++++++ .../test_tool_call_cost_tracking.py | 36 +++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f0424f674e2..dc9545d3933 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48600,6 +48600,11 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "supported_endpoints": [ "/v1/responses" ], @@ -48633,6 +48638,11 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "supported_endpoints": [ "/v1/responses" ], @@ -48666,6 +48676,11 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "supported_endpoints": [ "/v1/responses" ], @@ -48844,6 +48859,11 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "supported_endpoints": [ "/v1/responses" ], @@ -48872,6 +48892,11 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "supported_endpoints": [ "/v1/responses" ], diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f0424f674e2..dc9545d3933 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48600,6 +48600,11 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "supported_endpoints": [ "/v1/responses" ], @@ -48633,6 +48638,11 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "supported_endpoints": [ "/v1/responses" ], @@ -48666,6 +48676,11 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "supported_endpoints": [ "/v1/responses" ], @@ -48844,6 +48859,11 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "supported_endpoints": [ "/v1/responses" ], @@ -48872,6 +48892,11 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "supported_endpoints": [ "/v1/responses" ], diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 9bdded94513..3ffc1ea1b73 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -620,6 +620,42 @@ def _openai_responses_with_web_search_calls(model, num_calls): ) +@pytest.mark.parametrize( + "model", + [ + "bedrock_mantle/openai.gpt-5.6-sol", + "bedrock_mantle/openai.gpt-5.6-terra", + "bedrock_mantle/openai.gpt-5.6-luna", + "bedrock_mantle/openai.gpt-5.5", + "bedrock_mantle/openai.gpt-5.4", + ], +) +def test_bedrock_mantle_native_web_search_priced_per_query(model, local_model_cost_map): + """Bedrock's server-side Web Search bills $12.00 per 1,000 queries in every Region that + offers it (AWS Pricing API usage type ``Bedrock-Websearch-Queries``, SKUs HTPA2KVTTT65JDW6, + YV3KSHNXJCCU32NJ and E6H6Z5RYDRVX7HF9, effective 2026-08-01). Bedrock counts each search, + not each request, so a Responses output carrying three ``web_search_call`` items must bill + three queries. Without ``search_context_cost_per_query`` the default fallback silently + returns $0 and every native search is free.""" + from litellm.types.utils import Usage + + per_query = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + assert per_query == 0.012 + + for num_calls in (1, 3): + response = _openai_responses_with_web_search_calls(model, num_calls=num_calls) + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=response, + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + custom_llm_provider="bedrock_mantle", + standard_built_in_tools_params=None, + ) + assert cost == pytest.approx(num_calls * per_query), ( + f"{model} must bill {num_calls} x ${per_query} for {num_calls} web search(es), got ${cost}" + ) + + def test_openai_responses_web_search_priced_per_call(local_model_cost_map): """ Regression for LIT-5013 bug 1: OpenAI reasoning models (gpt-5 family, o-series, deep-research) From 9fcb5df9a374402f97e723e18b8592ab7e04acaf Mon Sep 17 00:00:00 2001 From: longwind48 Date: Sun, 23 Aug 2026 17:11:37 +0800 Subject: [PATCH 4/9] refactor(bedrock_mantle): revert the drive-by reformat of the tests The web_search work reformatted roughly 85 lines of unrelated tests in this file, reflowing assertions to 120 columns and normalising quotes, because ruff format was run over it and make format only covers litellm/. Rebuild the file from the merge base so the only deletions are the six web_search -> file_search retargets the change actually needs. While here: type the new helpers, make _WEB_SEARCH_CAPABLE_MODELS a Final tuple, drop the per-Region SKU hashes from the pricing test docstring since they go stale and a reader cannot act on them, and move search_context_cost_per_query into each entry's cost block ahead of litellm_provider, where every other entry carrying it puts it. Adds the one test the review found missing: that a web_search_call item and its url_citation annotations survive the response transform, which the config inherits rather than overrides. --- ...odel_prices_and_context_window_backup.json | 50 ++--- model_prices_and_context_window.json | 50 ++--- .../test_tool_call_cost_tracking.py | 11 +- ...bedrock_mantle_responses_transformation.py | 201 +++++++++++++++--- 4 files changed, 225 insertions(+), 87 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dc9545d3933..6481b6a6afa 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48594,17 +48594,17 @@ "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "search_context_cost_per_query": { - "search_context_size_high": 0.012, - "search_context_size_low": 0.012, - "search_context_size_medium": 0.012 - }, "supported_endpoints": [ "/v1/responses" ], @@ -48632,17 +48632,17 @@ "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "search_context_cost_per_query": { - "search_context_size_high": 0.012, - "search_context_size_low": 0.012, - "search_context_size_medium": 0.012 - }, "supported_endpoints": [ "/v1/responses" ], @@ -48670,17 +48670,17 @@ "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "search_context_cost_per_query": { - "search_context_size_high": 0.012, - "search_context_size_low": 0.012, - "search_context_size_medium": 0.012 - }, "supported_endpoints": [ "/v1/responses" ], @@ -48853,17 +48853,17 @@ "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, "output_cost_per_token": 3.3e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "search_context_cost_per_query": { - "search_context_size_high": 0.012, - "search_context_size_low": 0.012, - "search_context_size_medium": 0.012 - }, "supported_endpoints": [ "/v1/responses" ], @@ -48886,17 +48886,17 @@ "input_cost_per_token": 2.75e-06, "cache_read_input_token_cost": 2.75e-07, "output_cost_per_token": 1.65e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "search_context_cost_per_query": { - "search_context_size_high": 0.012, - "search_context_size_low": 0.012, - "search_context_size_medium": 0.012 - }, "supported_endpoints": [ "/v1/responses" ], diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dc9545d3933..6481b6a6afa 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48594,17 +48594,17 @@ "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "search_context_cost_per_query": { - "search_context_size_high": 0.012, - "search_context_size_low": 0.012, - "search_context_size_medium": 0.012 - }, "supported_endpoints": [ "/v1/responses" ], @@ -48632,17 +48632,17 @@ "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "search_context_cost_per_query": { - "search_context_size_high": 0.012, - "search_context_size_low": 0.012, - "search_context_size_medium": 0.012 - }, "supported_endpoints": [ "/v1/responses" ], @@ -48670,17 +48670,17 @@ "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "search_context_cost_per_query": { - "search_context_size_high": 0.012, - "search_context_size_low": 0.012, - "search_context_size_medium": 0.012 - }, "supported_endpoints": [ "/v1/responses" ], @@ -48853,17 +48853,17 @@ "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, "output_cost_per_token": 3.3e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "search_context_cost_per_query": { - "search_context_size_high": 0.012, - "search_context_size_low": 0.012, - "search_context_size_medium": 0.012 - }, "supported_endpoints": [ "/v1/responses" ], @@ -48886,17 +48886,17 @@ "input_cost_per_token": 2.75e-06, "cache_read_input_token_cost": 2.75e-07, "output_cost_per_token": 1.65e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.012, + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012 + }, "litellm_provider": "bedrock_mantle", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "search_context_cost_per_query": { - "search_context_size_high": 0.012, - "search_context_size_low": 0.012, - "search_context_size_medium": 0.012 - }, "supported_endpoints": [ "/v1/responses" ], diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 3ffc1ea1b73..f4604c3205a 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -631,12 +631,11 @@ def _openai_responses_with_web_search_calls(model, num_calls): ], ) def test_bedrock_mantle_native_web_search_priced_per_query(model, local_model_cost_map): - """Bedrock's server-side Web Search bills $12.00 per 1,000 queries in every Region that - offers it (AWS Pricing API usage type ``Bedrock-Websearch-Queries``, SKUs HTPA2KVTTT65JDW6, - YV3KSHNXJCCU32NJ and E6H6Z5RYDRVX7HF9, effective 2026-08-01). Bedrock counts each search, - not each request, so a Responses output carrying three ``web_search_call`` items must bill - three queries. Without ``search_context_cost_per_query`` the default fallback silently - returns $0 and every native search is free.""" + """Bedrock's server-side Web Search bills $12.00 per 1,000 queries, the same in every Region + that offers it (AWS Pricing API usage type ``Bedrock-Websearch-Queries``, effective + 2026-08-01). Bedrock counts each search rather than each request, so a Responses output + carrying three ``web_search_call`` items must bill three queries. Without + ``search_context_cost_per_query`` the default fallback silently returns $0.""" from litellm.types.utils import Usage per_query = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 87177ae07fb..07ef6e44714 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -8,6 +8,7 @@ gate, the URL construction for both paths, and the shared Bearer auth. """ import copy +from typing import Final import pytest @@ -46,7 +47,10 @@ class TestBedrockMantleResponsesURL: api_base="https://bedrock-mantle.us-east-2.api.aws/v1/", litellm_params={}, ) - assert url_trailing == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" + assert ( + url_trailing + == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" + ) def test_url_does_not_double_openai_v1(self, monkeypatch): monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) @@ -106,7 +110,9 @@ class TestBedrockMantleResponsesURL: with pytest.raises(ValueError, match="api\\.aws\\.attacker\\.example/'\\. Region names must contain only"): cfg.get_complete_url( api_base=None, - litellm_params={"aws_region_name": "us-east-1.api.aws.attacker.example/"}, + litellm_params={ + "aws_region_name": "us-east-1.api.aws.attacker.example/" + }, ) def test_url_region_default_us_east_1(self, monkeypatch): @@ -146,6 +152,7 @@ class TestBedrockMantleResponsesURL: assert url == "https://bedrock-mantle.us-east-2.api.aws/v1/responses" assert url.count("/responses") == 1 + def test_url_aws_region_name_overrides_stale_api_base(self, monkeypatch): monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) @@ -159,7 +166,9 @@ class TestBedrockMantleResponsesURL: class TestBedrockMantleGetLlmProviderRegion: - def test_get_llm_provider_uses_supplemental_litellm_params(self, monkeypatch, local_cost_map): + def test_get_llm_provider_uses_supplemental_litellm_params( + self, monkeypatch, local_cost_map + ): monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) @@ -176,7 +185,9 @@ class TestBedrockMantleGetLlmProviderRegion: # the resolved chat base) is on the /openai/v1 base per the AWS card. assert api_base == "https://bedrock-mantle.us-east-2.api.aws/openai/v1" - def test_get_llm_provider_uses_aws_region_from_litellm_params(self, monkeypatch, local_cost_map): + def test_get_llm_provider_uses_aws_region_from_litellm_params( + self, monkeypatch, local_cost_map + ): monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) @@ -210,14 +221,18 @@ class TestBedrockMantleResponsesAuth: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-key") monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) + headers = cfg.validate_environment( + headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() + ) assert headers["Authorization"] == "Bearer env-key" def test_bedrock_bearer_token_fallback(self, monkeypatch): monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "bearer-key") cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) + headers = cfg.validate_environment( + headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() + ) assert headers["Authorization"] == "Bearer bearer-key" def test_missing_bearer_does_not_raise_in_validate_environment(self, monkeypatch): @@ -225,7 +240,9 @@ class TestBedrockMantleResponsesAuth: monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) + headers = cfg.validate_environment( + headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() + ) assert "Authorization" not in headers def test_project_id_sets_openai_project_header(self): @@ -233,7 +250,9 @@ class TestBedrockMantleResponsesAuth: headers = cfg.validate_environment( headers={}, model="openai.gpt-5.5", - litellm_params=GenericLiteLLMParams(api_key="fake-key", aws_bedrock_project_id="proj_abc123def456"), + litellm_params=GenericLiteLLMParams( + api_key="fake-key", aws_bedrock_project_id="proj_abc123def456" + ), ) assert headers["OpenAI-Project"] == "proj_abc123def456" @@ -334,7 +353,9 @@ class TestBedrockMantleResponsesTools: from unittest.mock import patch cfg = BedrockMantleResponsesAPIConfig() - with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning") as mock_warning: + with patch( + "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning" + ) as mock_warning: cfg.map_openai_params( response_api_optional_params={"tools": [{"type": "file_search"}]}, model="openai.gpt-5.5", @@ -344,28 +365,47 @@ class TestBedrockMantleResponsesTools: assert "file_search" in str(mock_warning.call_args) -def _web_search_tool(external_web_access=False): +def _web_search_tool(external_web_access: bool = False) -> dict[str, object]: return {"type": "web_search", "external_web_access": external_web_access} -def _function_tool(): +def _function_tool() -> dict[str, object]: return {"type": "function", "name": "exec_command"} -def _codex_additional_tools_input(tools): +def _output_item_type(item: object) -> object: + return item.get("type") if isinstance(item, dict) else getattr(item, "type", None) + + +def _annotations_of_first_message(response: object) -> list[object]: + """Annotations off the first output_text block, tolerating dict or model output items.""" + for item in getattr(response, "output", []): + if _output_item_type(item) != "message": + continue + blocks = item.get("content", []) if isinstance(item, dict) else getattr(item, "content", []) + for block in blocks: + annotations = ( + block.get("annotations") if isinstance(block, dict) else getattr(block, "annotations", None) + ) + if annotations is not None: + return [a if isinstance(a, dict) else a.model_dump() for a in annotations] + return [] + + +def _codex_additional_tools_input(tools: list[dict[str, object]]) -> list[dict[str, object]]: return [ {"type": "additional_tools", "role": "developer", "tools": tools}, {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, ] -_WEB_SEARCH_CAPABLE_MODELS = [ +_WEB_SEARCH_CAPABLE_MODELS: Final = ( "openai.gpt-5.6-sol", "openai.gpt-5.6-terra", "openai.gpt-5.6-luna", "openai.gpt-5.5", "openai.gpt-5.4", -] +) class TestBedrockMantleResponsesNativeWebSearch: @@ -439,6 +479,65 @@ class TestBedrockMantleResponsesNativeWebSearch: for model in _WEB_SEARCH_CAPABLE_MODELS ) + def test_search_results_and_citations_survive_the_response_transform(self): + """Mantle returns Web Search results in the OpenAI Responses shape, so the config adds no + response-side handling. Lock that: a `web_search_call` item and the `url_citation` + annotations that make the answer attributable must both reach the caller intact.""" + import httpx + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + + citation: Final = { + "type": "url_citation", + "title": "Web Search - Amazon Bedrock", + "url": "https://docs.aws.amazon.com/bedrock/latest/userguide/web-search.html", + "start_index": 0, + "end_index": 12, + } + upstream: Final = { + "id": "resp_1", + "created_at": 0, + "model": "openai.gpt-5.6-terra", + "object": "response", + "output": [ + { + "id": "ws_1", + "type": "web_search_call", + "status": "completed", + "action": {"type": "search", "queries": ["bedrock web search regions"]}, + }, + { + "id": "msg_1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Three Regions", "annotations": [citation]}], + }, + ], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + } + + cfg = BedrockMantleResponsesAPIConfig() + out = cfg.transform_response_api_response( + model="openai.gpt-5.6-terra", + raw_response=httpx.Response(200, json=upstream), + logging_obj=LiteLLMLoggingObj( + model="openai.gpt-5.6-terra", + messages=[], + stream=False, + call_type="aresponses", + start_time=0, + litellm_call_id="1", + function_id="1", + ), + ) + + search_calls = [item for item in out.output if _output_item_type(item) == "web_search_call"] + assert len(search_calls) == 1, f"web_search_call item was dropped: {out.output}" + assert _annotations_of_first_message(out) == [citation] + def _codex_exec_tool(): return { @@ -515,7 +614,9 @@ class TestBedrockMantleServiceTier: from unittest.mock import patch cfg = BedrockMantleResponsesAPIConfig() - with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning") as mock_warning: + with patch( + "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning" + ) as mock_warning: cfg.map_openai_params( response_api_optional_params={"service_tier": "priority"}, model="openai.gpt-5.5", @@ -684,7 +785,9 @@ class TestBedrockMantleCodexAdditionalTools: def test_hoist_is_logged_at_debug_level(self): from unittest.mock import patch - with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.debug") as mock_debug: + with patch( + "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.debug" + ) as mock_debug: self._transform( input=[ {"type": "additional_tools", "role": "developer", "tools": self._CODEX_TOOLS}, @@ -819,7 +922,9 @@ class TestBedrockMantleResponsesRegistry: ) assert cfg is None - def test_price_map_flag_routes_non_gpt_name_to_openai_path(self, restore_model_cost): + def test_price_map_flag_routes_non_gpt_name_to_openai_path( + self, restore_model_cost + ): # Data-driven onboarding: a frontier model whose name does NOT match the # openai.gpt- convention can still be routed to /openai/v1/responses by # declaring use_openai_responses_path in its price-map entry, with no code @@ -845,8 +950,18 @@ class TestBedrockMantleResponsesRegistry: def test_gpt_5_5_price_map_declares_openai_responses_path(self, local_cost_map): # The gpt-5.x entries must carry the data-driven flag so frontier routing # does not rely on the name-string fallback alone. - assert litellm.model_cost["bedrock_mantle/openai.gpt-5.5"].get("use_openai_responses_path") is True - assert litellm.model_cost["bedrock_mantle/openai.gpt-5.4"].get("use_openai_responses_path") is True + assert ( + litellm.model_cost["bedrock_mantle/openai.gpt-5.5"].get( + "use_openai_responses_path" + ) + is True + ) + assert ( + litellm.model_cost["bedrock_mantle/openai.gpt-5.4"].get( + "use_openai_responses_path" + ) + is True + ) @pytest.mark.parametrize( "model", @@ -881,7 +996,9 @@ class TestBedrockMantleResponsesRegistry: ) assert cfg is None - def test_declared_responses_non_openai_routes_to_standard_path(self, restore_model_cost): + def test_declared_responses_non_openai_routes_to_standard_path( + self, restore_model_cost + ): # New feature: a non-OpenAI model declared mode=responses (e.g. via a # user's proxy model_info block) must route to the STANDARD /v1/responses # path, not the frontier /openai/v1/responses path. Fails before the @@ -989,7 +1106,11 @@ class TestMantleBaseSegment: ), ( "google.gemma-4-31b", - {"bedrock_mantle/google.gemma-4-31b": {"use_openai_responses_path": True}}, + { + "bedrock_mantle/google.gemma-4-31b": { + "use_openai_responses_path": True + } + }, "openai/v1", ), ( @@ -1028,7 +1149,11 @@ class TestMantleSupportsResponses: # chat-only supported_endpoints -> not supported (the discriminator) ( "openai.gpt-oss-safeguard-120b", - {"bedrock_mantle/openai.gpt-oss-safeguard-120b": {"supported_endpoints": ["/v1/chat/completions"]}}, + { + "bedrock_mantle/openai.gpt-oss-safeguard-120b": { + "supported_endpoints": ["/v1/chat/completions"] + } + }, False, ), # mode=responses (no supported_endpoints) -> supported @@ -1067,7 +1192,9 @@ class TestBedrockMantlePerModelResponsesURL: model=model, ) assert isinstance(cfg, BedrockMantleResponsesAPIConfig) - return cfg.get_complete_url(api_base=None, litellm_params={"aws_region_name": region}) + return cfg.get_complete_url( + api_base=None, litellm_params={"aws_region_name": region} + ) def test_gpt_oss_uses_standard_responses_path(self, local_cost_map): url = self._url_for("openai.gpt-oss-120b") @@ -1166,7 +1293,9 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) signer = BaseAWSLLM() - signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) + signer.get_credentials = MagicMock( + side_effect=AssertionError("get_credentials must not run for bearer auth") + ) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, signed_body = cfg.sign_request( @@ -1188,7 +1317,9 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-bearer") signer = BaseAWSLLM() - signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) + signer.get_credentials = MagicMock( + side_effect=AssertionError("get_credentials must not run for bearer auth") + ) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, _ = cfg.sign_request( @@ -1210,7 +1341,9 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-bearer") signer = BaseAWSLLM() - signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) + signer.get_credentials = MagicMock( + side_effect=AssertionError("get_credentials must not run for bearer auth") + ) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, _ = cfg.sign_request( @@ -1356,7 +1489,9 @@ class TestBedrockMantleResponsesSigV4: } cfg = BedrockMantleResponsesAPIConfig(aws_signer=BaseAWSLLM()) url = cfg.get_complete_url(api_base=None, litellm_params=params) - assert url == "https://bedrock-mantle.ap-southeast-2.api.aws/openai/v1/responses" + assert ( + url == "https://bedrock-mantle.ap-southeast-2.api.aws/openai/v1/responses" + ) headers, _ = cfg.sign_request( headers={}, @@ -1367,7 +1502,9 @@ class TestBedrockMantleResponsesSigV4: ) assert "/ap-southeast-2/bedrock/aws4_request" in headers["Authorization"] - def test_injected_default_region_base_does_not_override_aws_region_name(self, monkeypatch): + def test_injected_default_region_base_does_not_override_aws_region_name( + self, monkeypatch + ): """2nd-round adversarial regression: responses/main.py auto-injects litellm_params.api_base = https://bedrock-mantle..api.aws/v1 (default region, ignoring aws_region_name). The config must still pin BOTH the URL host @@ -1453,7 +1590,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=NoCredentialsError()) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError, match="Bedrock Mantle auth failed: no Bearer token and no usable") as exc: + with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -1483,7 +1620,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=cred_error) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError, match="Bedrock Mantle auth failed: no Bearer token and no usable") as exc: + with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -1508,7 +1645,9 @@ class TestBedrockMantleResponsesSigV4: signer = BaseAWSLLM() signer.get_credentials = MagicMock( - side_effect=ConnectTimeoutError(endpoint_url="https://sts.us-east-2.amazonaws.com") + side_effect=ConnectTimeoutError( + endpoint_url="https://sts.us-east-2.amazonaws.com" + ) ) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) From f4f84cc8d112da83592388d44ecb41cce005a929 Mon Sep 17 00:00:00 2001 From: longwind48 Date: Sun, 23 Aug 2026 17:35:46 +0800 Subject: [PATCH 5/9] fix(cost): bill web search off the count the provider reports _count_web_search_calls counts every web_search_call output item. Bedrock emits one such item per operation and Web Search has two, Search and Fetch, so a response that searched twice and opened one cached page billed three queries where Bedrock charges two. Measured against bedrock-mantle.us-east-1.api.aws across eleven responses, the count Bedrock reports as tool_usage.web_search.num_requests matched the number of search items every time, never the item total, and never the number of queries carried inside those items. Believe that count when a Responses payload carries one, and keep counting items when it does not, so OpenAI and Azure are unaffected. This is the same shape as the gemini, anthropic and xai paths, which already price off a provider-reported request count rather than off the returned items. The count is read through a Pydantic model rather than raw dict indexing, and both models allow extra keys so a payload that also reports other server-side tools still validates. --- .../llm_cost_calc/tool_call_cost_tracking.py | 29 ++++++- litellm/types/llms/openai.py | 19 +++++ .../test_tool_call_cost_tracking.py | 82 +++++++++++++++++++ 3 files changed, 127 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 887f167c262..fadd95ba2bc 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -5,11 +5,14 @@ Helper utilities for tracking the cost of built-in tools. from collections.abc import Mapping from typing import Any, Final, Literal +from pydantic import ValidationError + import litellm from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS from litellm.litellm_core_utils.llm_cost_calc.utils import _get_web_search_requests from litellm.types.llms.openai import ( FileSearchTool, + ProviderToolUsage, ResponsesAPIResponse, WebSearchOptions, ) @@ -137,16 +140,36 @@ class StandardBuiltInToolCostTracking: ) return per_call_cost * StandardBuiltInToolCostTracking._count_web_search_calls(response_object) + @staticmethod + def _reported_web_search_requests(response_object: object) -> int | None: + """The provider's own billable search count off a Responses payload, when it reports one. + + Bedrock returns ``tool_usage.web_search.num_requests``, which counts only the searches it + charges for and excludes the cached-page fetches that share the ``web_search_call`` item + type. Counting items instead overcharges every request that opened a page. + """ + tool_usage: Final = getattr(response_object, "tool_usage", None) + if tool_usage is None: + return None + try: + return ProviderToolUsage.model_validate(tool_usage).web_search.num_requests + except ValidationError: + return None + @staticmethod def _count_web_search_calls(response_object: object) -> int: """ Number of web searches to bill for on the per-call pricing path. Providers that report a request count in usage (gemini, anthropic, xai, vertex) are handled by - get_cost_for_web_search_request and never reach here. This path prices per call, so it must count - the web_search_call items. Chat-completions responses only expose url_citation annotations with no - count, so they floor to a single billable search. + get_cost_for_web_search_request and never reach here. Of the rest, some report the count on the + response itself and are believed over the item count; otherwise count the web_search_call items. + Chat-completions responses only expose url_citation annotations with no count, so they floor to a + single billable search. """ + reported: Final = StandardBuiltInToolCostTracking._reported_web_search_requests(response_object) + if reported is not None: + return reported if isinstance(response_object, ResponsesAPIResponse): count = sum( 1 for output_item in response_object.output if _output_item_type(output_item) == "web_search_call" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index e7a3f825455..8dce1e7be32 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1300,6 +1300,25 @@ One of: completed, failed, in_progress, cancelled, queued, or incomplete. """ +class WebSearchToolUsage(BaseModel): + num_requests: int + + model_config = ConfigDict(extra="allow") + + +class ProviderToolUsage(BaseModel): + """A Responses payload's own `tool_usage`, used to bill server-side tools off the provider's + count rather than off the returned items. Bedrock populates `web_search.num_requests`. + + Extra keys are allowed on both models so a payload that also reports other server-side tools + still validates and the web search count is still read. + """ + + web_search: WebSearchToolUsage + + model_config = ConfigDict(extra="allow") + + class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): id: str created_at: int diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index f4604c3205a..2cc865583c8 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -620,6 +620,88 @@ def _openai_responses_with_web_search_calls(model, num_calls): ) +def _responses_with_reported_web_search(model, num_requests, search_items, fetch_items): + """A Bedrock-shaped Responses payload: `web_search_call` items for both operations, plus the + `tool_usage.web_search.num_requests` count Bedrock reports for the searches it charges for.""" + from litellm.types.llms.openai import ResponsesAPIResponse + + output = [ + { + "id": f"ws_s{i}", + "type": "web_search_call", + "status": "completed", + "action": {"type": "search", "queries": ["a", "b", "c"]}, + } + for i in range(search_items) + ] + [ + { + "id": f"ws_f{i}", + "type": "web_search_call", + "status": "completed", + "action": {"type": "open_page", "url": "https://example.invalid/page"}, + } + for i in range(fetch_items) + ] + return ResponsesAPIResponse( + id="resp_1", + created_at=0, + model=model, + object="response", + output=output, + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + tool_usage={"web_search": {"num_requests": num_requests}}, + ) + + +def test_bedrock_mantle_web_search_bills_the_count_bedrock_reports(local_model_cost_map): + """Bedrock reports its billable search count as ``tool_usage.web_search.num_requests`` and it + excludes cached-page fetches, which arrive as ``web_search_call`` items indistinguishable from + searches by item type. Measured against bedrock-mantle.us-east-1.api.aws: a response with five + search items and one ``open_page`` item reports 5, and one with one search and one + ``open_page`` reports 1. Counting items would bill 6 and 2, overcharging every request that + opened a page. Three queries inside a single search item still count as one request.""" + from litellm.types.utils import Usage + + model = "bedrock_mantle/openai.gpt-5.6-terra" + per_query = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + + for num_requests, search_items, fetch_items in ((5, 5, 1), (1, 1, 1), (2, 2, 1)): + response = _responses_with_reported_web_search(model, num_requests, search_items, fetch_items) + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=response, + usage=usage, + custom_llm_provider="bedrock_mantle", + standard_built_in_tools_params=None, + ) + assert cost == pytest.approx(num_requests * per_query), ( + f"{search_items + fetch_items} items reporting {num_requests} requests must bill " + f"{num_requests} x ${per_query}, got ${cost}" + ) + + +def test_web_search_falls_back_to_counting_items_when_no_count_is_reported(local_model_cost_map): + """Providers that report nothing must keep the item-count behaviour, so adding the Bedrock path + cannot change what OpenAI and Azure bill.""" + model = "gpt-5-nano" + response = _openai_responses_with_web_search_calls(model, num_calls=3) + assert not hasattr(response, "tool_usage") + assert StandardBuiltInToolCostTracking._count_web_search_calls(response) == 3 + + +def test_malformed_reported_web_search_count_falls_back_to_items(local_model_cost_map): + """A tool_usage payload that does not carry an integer count must not raise or bill zero.""" + response = _responses_with_reported_web_search("bedrock_mantle/openai.gpt-5.6-terra", 2, 2, 1) + response.tool_usage = {"web_search": {"num_requests": "not-a-number"}} + assert StandardBuiltInToolCostTracking._count_web_search_calls(response) == 3 + + response.tool_usage = {"unexpected_shape": True} + assert StandardBuiltInToolCostTracking._count_web_search_calls(response) == 3 + + @pytest.mark.parametrize( "model", [ From 18e74ba1cb08f6cb304a87e9ab3ee7f059905880 Mon Sep 17 00:00:00 2001 From: longwind48 Date: Sun, 23 Aug 2026 18:13:25 +0800 Subject: [PATCH 6/9] refactor(cost): tighten the reported web search count and its tests Review follow-ups, no behaviour change to the billed amount except the zero case, which is now covered. Rename ProviderToolUsage to ResponsesToolUsage: it models a field on a Responses payload, not a provider-level concept. Record why it is read as an extra rather than declared on ResponsesAPIResponse: declaring it makes any tool_usage shape we do not model fail the whole response, and a payload that reports a different server-side tool is a shape we should expect. Tests: bill a turn that fetched a cached page without searching, where Bedrock reports zero and the item path would floor at one. Pin the streaming path, where the cost calculator sees the response unwrapped out of the terminal ResponseCompletedEvent and the event alone would floor the bill. Stop mutating a constructed model to build the unreadable-tool_usage cases, take the raw block as a factory argument instead, and cover a bare string and None as well. Drop a third copy of _output_item_type and an over-built annotation walker in favour of asserting against model_dump(). --- .../llm_cost_calc/tool_call_cost_tracking.py | 4 +- litellm/types/llms/openai.py | 7 +- .../test_tool_call_cost_tracking.py | 90 ++++++++++++++++--- ...bedrock_mantle_responses_transformation.py | 25 +----- 4 files changed, 85 insertions(+), 41 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index fadd95ba2bc..a4fbd04b8ee 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -12,8 +12,8 @@ from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS from litellm.litellm_core_utils.llm_cost_calc.utils import _get_web_search_requests from litellm.types.llms.openai import ( FileSearchTool, - ProviderToolUsage, ResponsesAPIResponse, + ResponsesToolUsage, WebSearchOptions, ) from litellm.types.utils import ( @@ -152,7 +152,7 @@ class StandardBuiltInToolCostTracking: if tool_usage is None: return None try: - return ProviderToolUsage.model_validate(tool_usage).web_search.num_requests + return ResponsesToolUsage.model_validate(tool_usage).web_search.num_requests except ValidationError: return None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 8dce1e7be32..6afe45e06c5 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1306,12 +1306,13 @@ class WebSearchToolUsage(BaseModel): model_config = ConfigDict(extra="allow") -class ProviderToolUsage(BaseModel): +class ResponsesToolUsage(BaseModel): """A Responses payload's own `tool_usage`, used to bill server-side tools off the provider's count rather than off the returned items. Bedrock populates `web_search.num_requests`. - Extra keys are allowed on both models so a payload that also reports other server-side tools - still validates and the web search count is still read. + Deliberately not a declared field on ``ResponsesAPIResponse``: it arrives as an extra and is + validated only where the cost path reads it, so a payload reporting a tool we do not model, or + a shape we do not expect, still parses instead of failing the whole response. """ web_search: WebSearchToolUsage diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 2cc865583c8..6c0635080c1 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -620,9 +620,9 @@ def _openai_responses_with_web_search_calls(model, num_calls): ) -def _responses_with_reported_web_search(model, num_requests, search_items, fetch_items): +def _responses_with_reported_web_search(model, search_items, fetch_items, tool_usage): """A Bedrock-shaped Responses payload: `web_search_call` items for both operations, plus the - `tool_usage.web_search.num_requests` count Bedrock reports for the searches it charges for.""" + raw `tool_usage` block, which Bedrock populates with the count it charges for.""" from litellm.types.llms.openai import ResponsesAPIResponse output = [ @@ -651,7 +651,7 @@ def _responses_with_reported_web_search(model, num_requests, search_items, fetch parallel_tool_calls=False, tool_choice="auto", tools=[], - tool_usage={"web_search": {"num_requests": num_requests}}, + tool_usage=tool_usage, ) @@ -668,8 +668,21 @@ def test_bedrock_mantle_web_search_bills_the_count_bedrock_reports(local_model_c per_query = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) - for num_requests, search_items, fetch_items in ((5, 5, 1), (1, 1, 1), (2, 2, 1)): - response = _responses_with_reported_web_search(model, num_requests, search_items, fetch_items) + cases = ( + {"searches": 5, "fetches": 1, "reported": 5}, + {"searches": 1, "fetches": 1, "reported": 1}, + {"searches": 2, "fetches": 1, "reported": 2}, + # A follow-up turn can fetch a cached page without searching, and Bedrock then charges + # nothing. The item path floors at one search; a reported count must not be floored. + {"searches": 0, "fetches": 2, "reported": 0}, + ) + for case in cases: + response = _responses_with_reported_web_search( + model, + search_items=case["searches"], + fetch_items=case["fetches"], + tool_usage={"web_search": {"num_requests": case["reported"]}}, + ) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( model=model, response_object=response, @@ -677,9 +690,9 @@ def test_bedrock_mantle_web_search_bills_the_count_bedrock_reports(local_model_c custom_llm_provider="bedrock_mantle", standard_built_in_tools_params=None, ) - assert cost == pytest.approx(num_requests * per_query), ( - f"{search_items + fetch_items} items reporting {num_requests} requests must bill " - f"{num_requests} x ${per_query}, got ${cost}" + assert cost == pytest.approx(case["reported"] * per_query), ( + f"{case['searches'] + case['fetches']} items reporting {case['reported']} requests must " + f"bill {case['reported']} x ${per_query}, got ${cost}" ) @@ -692,13 +705,62 @@ def test_web_search_falls_back_to_counting_items_when_no_count_is_reported(local assert StandardBuiltInToolCostTracking._count_web_search_calls(response) == 3 -def test_malformed_reported_web_search_count_falls_back_to_items(local_model_cost_map): - """A tool_usage payload that does not carry an integer count must not raise or bill zero.""" - response = _responses_with_reported_web_search("bedrock_mantle/openai.gpt-5.6-terra", 2, 2, 1) - response.tool_usage = {"web_search": {"num_requests": "not-a-number"}} - assert StandardBuiltInToolCostTracking._count_web_search_calls(response) == 3 +def test_streamed_web_search_bills_the_reported_count(local_model_cost_map): + """On the streaming path the cost calculator is handed the response unwrapped out of the + terminal `ResponseCompletedEvent`, not the event. Pin that: the reported count has to survive + the unwrap, because the event itself carries no output items and would floor the bill at one + search instead of charging what Bedrock reported.""" + import datetime - response.tool_usage = {"unexpected_shape": True} + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.llms.openai import ResponseCompletedEvent + from litellm.types.utils import Usage + + model = "bedrock_mantle/openai.gpt-5.6-terra" + per_query = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + response = _responses_with_reported_web_search( + model, search_items=2, fetch_items=3, tool_usage={"web_search": {"num_requests": 2}} + ) + + logging_obj = LiteLLMLoggingObj( + model=model, messages=[], stream=True, call_type="aresponses", + start_time=0, litellm_call_id="1", function_id="1", + ) + now = datetime.datetime.now() + assembled = logging_obj._get_assembled_streaming_response( + result=ResponseCompletedEvent(type="response.completed", response=response), + start_time=now, end_time=now, is_async=True, streaming_chunks=[], + ) + assert assembled is response, "the completed event must be unwrapped to the response itself" + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=assembled, + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + custom_llm_provider="bedrock_mantle", + standard_built_in_tools_params=None, + ) + assert cost == pytest.approx(2 * per_query), ( + f"5 items reporting 2 requests must bill 2 x ${per_query} after the stream is assembled, got ${cost}" + ) + + +@pytest.mark.parametrize( + "tool_usage", + [ + {"web_search": {"num_requests": "not-a-number"}}, + {"unexpected_shape": True}, + "not-an-object", + None, + ], +) +def test_unreadable_reported_web_search_count_falls_back_to_items(tool_usage, local_model_cost_map): + """A tool_usage block we cannot read must neither raise nor bill zero: fall back to the items. + Reading it defensively here, rather than declaring it on the response model, is what keeps a + payload that reports an unmodelled tool from failing the whole response.""" + response = _responses_with_reported_web_search( + "bedrock_mantle/openai.gpt-5.6-terra", search_items=2, fetch_items=1, tool_usage=tool_usage + ) assert StandardBuiltInToolCostTracking._count_web_search_calls(response) == 3 diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 07ef6e44714..39f027789d3 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -373,25 +373,6 @@ def _function_tool() -> dict[str, object]: return {"type": "function", "name": "exec_command"} -def _output_item_type(item: object) -> object: - return item.get("type") if isinstance(item, dict) else getattr(item, "type", None) - - -def _annotations_of_first_message(response: object) -> list[object]: - """Annotations off the first output_text block, tolerating dict or model output items.""" - for item in getattr(response, "output", []): - if _output_item_type(item) != "message": - continue - blocks = item.get("content", []) if isinstance(item, dict) else getattr(item, "content", []) - for block in blocks: - annotations = ( - block.get("annotations") if isinstance(block, dict) else getattr(block, "annotations", None) - ) - if annotations is not None: - return [a if isinstance(a, dict) else a.model_dump() for a in annotations] - return [] - - def _codex_additional_tools_input(tools: list[dict[str, object]]) -> list[dict[str, object]]: return [ {"type": "additional_tools", "role": "developer", "tools": tools}, @@ -534,9 +515,9 @@ class TestBedrockMantleResponsesNativeWebSearch: ), ) - search_calls = [item for item in out.output if _output_item_type(item) == "web_search_call"] - assert len(search_calls) == 1, f"web_search_call item was dropped: {out.output}" - assert _annotations_of_first_message(out) == [citation] + dumped = out.model_dump() + assert [item["type"] for item in dumped["output"]] == ["web_search_call", "message"] + assert dumped["output"][1]["content"][0]["annotations"] == [citation] def _codex_exec_tool(): From 836283bdb7b3a447d44e3df0e12fe13fe48dff70 Mon Sep 17 00:00:00 2001 From: longwind48 Date: Tue, 25 Aug 2026 13:45:10 +0800 Subject: [PATCH 7/9] test(cost): assert the item-count fallback through the public cost entry point Two fallback tests reached for _count_web_search_calls directly. Bill them through get_cost_for_built_in_tools instead: same guarantee, and the assertion is now the charge a caller would see rather than an internal count. --- .../test_bedrock_mantle_responses_transformation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 39f027789d3..2ed12141f1d 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -425,11 +425,11 @@ class TestBedrockMantleResponsesNativeWebSearch: assert params["tools"] == [_function_tool()] def test_web_search_is_not_advertised_unless_the_request_asks_for_it(self, local_model_cost_map): - """The tool type widens only for requests that carry it, so a capable model's - other requests keep the provider-wide set and never reach the cost map.""" + """The tool type set widens only for requests that carry a Web Search tool, so a capable + model's other requests keep the provider-wide set and never reach the cost map.""" cfg = BedrockMantleResponsesAPIConfig() supported = cfg._supported_response_tool_types(tools=[_function_tool()], model="openai.gpt-5.6-terra") - assert "web_search" not in supported + assert "web_search" not in supported, "a request carrying no web_search tool must not widen the set" assert "function" in supported def test_web_search_hoisted_out_of_codex_additional_tools(self, local_model_cost_map): From 32095c5f52423e53d46f93fe49a63577fc613179 Mon Sep 17 00:00:00 2001 From: longwind48 Date: Fri, 28 Aug 2026 11:13:28 +0800 Subject: [PATCH 8/9] fix(guardrails): enforce allowlists on built-in responses tools Built-in server-side tools such as web_search carry no name of their own, only a type, so the Responses tool-name extractor returned nothing for them. Both consumers of that extractor, the key/team allowed_tools check in auth and ToolPolicyGuardrail, are name-based, so a key with a restrictive allowlist or a default-deny tool policy could still send web_search, reach the internet, and bill for it. Surface built-in tools under their type, matching what the Anthropic messages path already does with its flat tools[].name. A missing function or custom name still yields nothing, so "function" can never stand in for the real tool. --- .../guardrail_translation/handler.py | 29 ++++++---- .../proxy/test_tools_allowlist_enforcement.py | 53 +++++++++++++++++++ 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 7c5d8ac99ad..c119e490ecd 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -216,18 +216,25 @@ class OpenAIResponsesHandler(BaseTranslation): return data + @staticmethod + def _request_tool_name(tool: object) -> str | None: + """The name a Responses request tool acts under: ``name`` for function and custom, + ``server_label`` for mcp, and the bare ``type`` for built-in server-side tools + (web_search, code_interpreter, ...) that carry no name of their own. Those bill and + reach the internet, so allowlist checks must see them under some name.""" + if not isinstance(tool, dict): + return None + tool_type: Final = tool.get("type") + if tool_type in ("function", "custom"): + name: Final = tool.get("name") + return str(name) if name else None + if tool_type == "mcp": + server_label: Final = tool.get("server_label") + return str(server_label) if server_label else None + return str(tool_type) if tool_type else None + def extract_request_tool_names(self, data: dict) -> list[str]: - """Extract tool names from Responses API request (tools[].name for function - and custom, tools[].server_label for mcp).""" - names: Final[list[str]] = [] - for tool in data.get("tools") or []: - if not isinstance(tool, dict): - continue - if tool.get("type") in ("function", "custom") and tool.get("name"): - names.append(str(tool["name"])) - elif tool.get("type") == "mcp" and tool.get("server_label"): - names.append(str(tool["server_label"])) - return names + return [name for tool in data.get("tools") or [] if (name := self._request_tool_name(tool)) is not None] def _extract_and_transform_tools( self, diff --git a/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py b/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py index 3e9c7c14b95..431cc4dc3cd 100644 --- a/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py +++ b/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py @@ -86,6 +86,31 @@ class TestExtractRequestToolNames: "get_current_weather", ] + def test_openai_responses_builtin_tools(self): + """Built-in server-side tools carry no name of their own, so they act under their + type; without that a restricted key could still reach the internet and bill for + web_search while its allowlist named nothing of the sort (VERIA finding on PR #37995).""" + data = { + "tools": [ + {"type": "web_search"}, + {"type": "code_interpreter", "container": {"type": "auto"}}, + {"type": "function", "name": "get_current_weather"}, + {"type": "mcp", "server_label": "dmcp", "server_url": "http://x"}, + ] + } + assert extract_request_tool_names("/v1/responses", data) == [ + "web_search", + "code_interpreter", + "get_current_weather", + "dmcp", + ] + + def test_openai_responses_unnamed_tool_yields_no_name(self): + """A function or custom tool missing its name must not fall back to the bare type: + that would let "function" satisfy an allowlist that never granted the real tool.""" + data = {"tools": [{"type": "function"}, {"type": "custom", "name": ""}, {"type": "mcp"}, "junk"]} + assert extract_request_tool_names("/v1/responses", data) == [] + def test_anthropic_tools(self): data = {"tools": [{"name": "get_weather"}, {"name": "run_sql"}]} assert extract_request_tool_names("/v1/messages", data) == [ @@ -227,6 +252,34 @@ class TestCheckToolsAllowlist: assert exc_info.value.type == ProxyErrorTypes.tool_access_denied assert "restricted_tool" in str(exc_info.value.message) + @pytest.mark.asyncio + async def test_disallowed_builtin_web_search_raises_on_responses_route(self): + token = _token(metadata={"allowed_tools": ["run_sql"]}) + body = {"tools": [{"type": "web_search"}]} + with pytest.raises(ProxyException) as exc_info: + await check_tools_allowlist( + request_body=body, + valid_token=token, + team_object=None, + route="/v1/responses", + ) + assert exc_info.value.type == ProxyErrorTypes.tool_access_denied + assert "web_search" in str(exc_info.value.message) + + @pytest.mark.asyncio + async def test_allowlisted_builtin_web_search_passes_on_responses_route(self): + token = _token(metadata={"allowed_tools": ["web_search", "run_sql"]}) + tools = [{"type": "web_search"}, {"type": "function", "name": "run_sql"}] + body = {"tools": tools} + assert extract_request_tool_names("/v1/responses", body) == ["web_search", "run_sql"] + await check_tools_allowlist( + request_body=body, + valid_token=token, + team_object=None, + route="/v1/responses", + ) + assert body["tools"] == tools + @pytest.mark.asyncio async def test_team_allowlist_used_when_key_empty(self): token = _token( From 53d7e124b7af16ed5eae6de8a3680ce17f941be5 Mon Sep 17 00:00:00 2001 From: longwind48 Date: Sat, 29 Aug 2026 11:13:34 +0800 Subject: [PATCH 9/9] fix(guardrails): extract tool names nested in additional_tools input items Codex's responses-lite wire mode declares tools inside an `additional_tools` input item rather than in top-level `tools`, and the Mantle transformation hoists them back out before dispatch. The key/team allowed_tools check and ToolPolicyGuardrail both read only `tools`, so a restricted key could place any tool, web_search included, in `input` and have it forwarded unchecked. Walk `input` for those items during extraction so nested tools face the same allowlist as declared ones. A missing or non-list `tools` slot, and a plain string `input`, yield nothing rather than raising on the auth hot path. --- .../guardrail_translation/handler.py | 22 +++++++- .../proxy/test_tools_allowlist_enforcement.py | 53 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index c119e490ecd..145ca05d6e5 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -233,8 +233,28 @@ class OpenAIResponsesHandler(BaseTranslation): return str(server_label) if server_label else None return str(tool_type) if tool_type else None + @staticmethod + def _tools_nested_in_input_item(item: object) -> tuple[object, ...]: + """Tools declared inside an ``additional_tools`` input item. Codex's responses-lite wire + mode ships tool definitions there instead of in top-level ``tools``, and providers hoist + them back out before dispatch, so reading only ``tools`` would miss them.""" + if not isinstance(item, dict) or item.get("type") != "additional_tools": + return () + tools: Final = item.get("tools") + return tuple(tools) if isinstance(tools, list) else () + def extract_request_tool_names(self, data: dict) -> list[str]: - return [name for tool in data.get("tools") or [] if (name := self._request_tool_name(tool)) is not None] + input_items: Final = data.get("input") + nested: Final = ( + tuple(tool for item in input_items for tool in self._tools_nested_in_input_item(item)) + if isinstance(input_items, list) + else () + ) + return [ + name + for tool in (*(data.get("tools") or []), *nested) + if (name := self._request_tool_name(tool)) is not None + ] def _extract_and_transform_tools( self, diff --git a/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py b/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py index 431cc4dc3cd..85358417494 100644 --- a/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py +++ b/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py @@ -105,6 +105,41 @@ class TestExtractRequestToolNames: "dmcp", ] + def test_openai_responses_tools_nested_in_additional_tools_input_item(self): + """Codex's responses-lite wire mode declares tools inside an `additional_tools` input + item, and providers hoist them into top-level `tools` before dispatch. Reading only + `tools` would let a restricted key smuggle any tool through `input` + (VERIA finding on PR #37995).""" + data = { + "input": [ + {"type": "message", "role": "user", "content": "hi"}, + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "web_search"}, {"type": "function", "name": "run_sql"}], + }, + ], + "tools": [{"type": "function", "name": "declared_up_front"}], + } + assert extract_request_tool_names("/v1/responses", data) == [ + "declared_up_front", + "web_search", + "run_sql", + ] + + def test_openai_responses_malformed_additional_tools_yields_no_name(self): + """An `additional_tools` item with a missing or non-list `tools` slot, and a plain string + input, must not raise on the auth hot path.""" + data = { + "input": [ + {"type": "additional_tools"}, + {"type": "additional_tools", "tools": "not-a-list"}, + "junk", + ] + } + assert extract_request_tool_names("/v1/responses", data) == [] + assert extract_request_tool_names("/v1/responses", {"input": "plain string"}) == [] + def test_openai_responses_unnamed_tool_yields_no_name(self): """A function or custom tool missing its name must not fall back to the bare type: that would let "function" satisfy an allowlist that never granted the real tool.""" @@ -280,6 +315,24 @@ class TestCheckToolsAllowlist: ) assert body["tools"] == tools + @pytest.mark.asyncio + async def test_disallowed_tool_nested_in_input_raises_on_responses_route(self): + token = _token(metadata={"allowed_tools": ["run_sql"]}) + body = { + "input": [ + {"type": "additional_tools", "role": "developer", "tools": [{"type": "web_search"}]}, + ] + } + with pytest.raises(ProxyException) as exc_info: + await check_tools_allowlist( + request_body=body, + valid_token=token, + team_object=None, + route="/v1/responses", + ) + assert exc_info.value.type == ProxyErrorTypes.tool_access_denied + assert "web_search" in str(exc_info.value.message) + @pytest.mark.asyncio async def test_team_allowlist_used_when_key_empty(self): token = _token(