diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 3bc1f713a42..6221af742c9 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -16,6 +16,8 @@ from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.openai import ResponsesAPIResponse if TYPE_CHECKING: + import httpx + from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse @@ -32,6 +34,12 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict): client: ReadOnly[object] +def _provider_response_headers(stream: object) -> "httpx.Headers | None": + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator + + return stream.response.headers if isinstance(stream, BaseResponsesAPIStreamingIterator) else None + + def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: """`responses()` runs `get_llm_provider()` itself, so hand back the prefixed model `completion()` started from.""" return f"{custom_llm_provider}/{model}" @@ -269,6 +277,7 @@ class ResponsesToCompletionBridgeHandler: model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, + _response_headers=_provider_response_headers(result), ) return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider) @@ -374,6 +383,7 @@ class ResponsesToCompletionBridgeHandler: model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, + _response_headers=_provider_response_headers(result), ) return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 2d43398a7a7..4bb7136743f 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -336,6 +336,28 @@ def reasoning_origin_for(provider: str | None, model: str) -> str | None: return f"{_REASONING_ORIGIN_PREFIX}bedrock/{get_bedrock_base_model(model)}:" +def _tool_reference_as_text(part: object) -> object: + match part: + case {"type": "tool_reference", "tool_name": str() as tool_name}: + return {"type": "text", "text": tool_name} + case _: + return part + + +def _tool_message_with_tool_references_as_text(msg: Mapping[str, object]) -> Mapping[str, object]: + content: Final = msg.get("content") + if msg.get("role") != "tool" or not isinstance(content, list): + return msg + parts: Final = cast("list[object]", content) # cast-ok: isinstance leaves the part type unknown + return {**msg, "content": [_tool_reference_as_text(part) for part in parts]} + + +def _tool_references_as_text(messages: Sequence["AllMessageValues"]) -> tuple["AllMessageValues", ...]: + return cast( # cast-ok: each message keeps its fields, and only tool content parts change + 'tuple["AllMessageValues", ...]', tuple(_tool_message_with_tool_references_as_text(msg) for msg in messages) + ) + + def _with_reasoning_origin(output_items: Sequence[object], origin: str | None) -> Sequence[object]: if origin is None: return output_items @@ -859,7 +881,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def convert_chat_completion_messages_to_responses_api( self, - messages: list["AllMessageValues"], + messages: Sequence["AllMessageValues"], *, drop_params: bool = False, keep_prompt_cache_breakpoints: bool = False, @@ -883,7 +905,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def _convert_chat_completion_messages_to_responses_input( self, - messages: list["AllMessageValues"], + messages: Sequence["AllMessageValues"], *, drop_params: bool = False, replay_tool_search: bool = False, @@ -1140,7 +1162,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): and supports_openai_prompt_cache_breakpoint(base_model, provider) ) converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api( - messages, + _tool_references_as_text(messages) if provider == "bedrock" else messages, drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params), keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint, replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index a55f9a08195..9bc71d8fe27 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1313,10 +1313,17 @@ def _bedrock_serves_responses(model: str) -> bool: def bedrock_responses_bridge_optional_params(optional_params: Mapping[str, object]) -> Mapping[str, object]: if optional_params.get("store") is not None: return optional_params - include: Final = optional_params.get("include") + extra_body: Final = optional_params.get("extra_body") + extra_body_params: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body) if isinstance(extra_body, dict) else {} + include: Final = extra_body_params.get("include", optional_params.get("include")) requested: Final = _INCLUDE_ADAPTER.validate_python(include) if isinstance(include, (list, tuple)) else () return { **optional_params, + **( + {"extra_body": {key: value for key, value in extra_body_params.items() if key != "include"}} + if "include" in extra_body_params + else {} + ), "store": False, "include": [*requested, *(() if _ENCRYPTED_REASONING in requested else (_ENCRYPTED_REASONING,))], } diff --git a/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py b/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py index 2fbd4e9aa20..7f5d91fec30 100644 --- a/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py +++ b/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py @@ -11,7 +11,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase( scenario="tool_search", litellm_endpoint="/v1/messages", litellm_request={ - "model": "bedrock/global.openai.gpt-6.1-sol", + "model": "bedrock/converse/global.openai.gpt-6.1-sol", "max_tokens": 1024, "thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}, @@ -139,7 +139,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase( "id": ANY, "type": "message", "role": "assistant", - "model": "bedrock/global.openai.gpt-6.1-sol", + "model": "bedrock/converse/global.openai.gpt-6.1-sol", "stop_sequence": None, "usage": {"input_tokens": 100, "output_tokens": 15}, "content": [{"type": "text", "text": "The web search tool is loaded and ready to use."}], diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index c60c780e7aa..894138447f8 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -1238,6 +1238,80 @@ async def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(loca assert "include" not in body +@pytest.mark.parametrize( + "include_param", + [ + {"include": ["file_search_call.results"]}, + {"extra_body": {"include": ["file_search_call.results"]}}, + ], +) +async def test_tools_with_reasoning_bridge_keeps_encrypted_reasoning_beside_a_requested_include( + local_cost_map, fake_aws_env, include_param +): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + client=client, + **include_param, + ) + + body = json.loads(requests[0].content) + assert body["store"] is False + assert body["include"] == ["file_search_call.results", "reasoning.encrypted_content"] + + +async def test_tools_with_reasoning_bridge_sends_a_tool_reference_result_as_its_tool_name_like_converse( + local_cost_map, fake_aws_env +): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[ + {"role": "user", "content": "find a web tool"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "ToolSearch", "arguments": "{}"}} + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "tool_reference", "tool_name": "WebSearch"}], + }, + ], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + client=client, + ) + + body = json.loads(requests[0].content) + tool_output = next(item for item in body["input"] if item["type"] == "function_call_output") + assert tool_output["output"] == [{"type": "input_text", "text": "WebSearch"}] + + +@pytest.mark.parametrize("stream", [False, True], ids=["plain", "streamed"]) +async def test_tools_with_reasoning_bridge_returns_the_provider_response_headers(local_cost_map, fake_aws_env, stream): + reply_kwargs: Final = ( + { + "content": _responses_stream_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"), + "headers": {"content-type": "text/event-stream", "x-amzn-requestid": "req-1"}, + } + if stream + else { + "json": _responses_reply_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"), + "headers": {"x-amzn-requestid": "req-1"}, + } + ) + _, response = await _bedrock_turn([{"role": "user", "content": "weather in Paris"}], **reply_kwargs) + + assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-1" + + async def test_tools_with_reasoning_and_a_param_the_bridge_drops_fail_like_converse_without_drop_params( local_cost_map, fake_aws_env ): diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts index 78372fe56d9..74e9a1d1f33 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts @@ -335,6 +335,38 @@ describe("ToolsSection utils", () => { expect(result[0].called).toBe(true); expect(result[0].callData?.arguments).toEqual({ location: "Tokyo" }); }); + + it("should parse tool calls from a Responses API response output", () => { + const log: Partial = { + request_id: "test-responses-1", + proxy_server_request: { + tools: [ + { type: "function", function: { name: "get_weather", description: "Get current weather" } }, + { type: "function", function: { name: "get_time", description: "Get current time" } }, + ], + }, + response: { + object: "response", + output: [ + { type: "reasoning", summary: [] }, + { + type: "function_call", + call_id: "call_abc", + name: "get_weather", + arguments: '{"location": "Paris"}', + }, + ], + }, + } as any; + + const result = parseToolsFromLog(log as LogEntry); + + expect(result.map((tool) => [tool.name, tool.called])).toEqual([ + ["get_weather", true], + ["get_time", false], + ]); + expect(result[0].callData).toEqual({ id: "call_abc", name: "get_weather", arguments: { location: "Paris" } }); + }); }); describe("hasTools", () => { diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts index 5e76a09c155..789fa4f0f88 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts @@ -80,31 +80,36 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] { return responseData.tool_calls; } + // Responses API format: response.output[].type === "function_call" + if (Array.isArray(responseData.output)) { + const toolCalls = toolCallsFromOutput(responseData.output); + if (toolCalls.length > 0) return toolCalls; + } + // Realtime API format: response.results[].response.output[].type === "function_call" if (Array.isArray(responseData.results)) { - const toolCalls: ToolCall[] = []; - for (const result of responseData.results) { - if (result.type === "response.done" && result.response?.output) { - for (const item of result.response.output) { - if (item.type === "function_call") { - toolCalls.push({ - id: item.call_id || "", - type: "function", - function: { - name: item.name || "", - arguments: item.arguments || "{}", - }, - }); - } - } - } - } + const toolCalls = responseData.results + .filter((result: any) => result.type === "response.done" && Array.isArray(result.response?.output)) + .flatMap((result: any) => toolCallsFromOutput(result.response.output)); if (toolCalls.length > 0) return toolCalls; } return []; } +function toolCallsFromOutput(output: any[]): ToolCall[] { + return output + .filter((item: any) => item.type === "function_call") + .map((item: any) => ({ + id: item.call_id || "", + type: "function", + function: { + name: item.name || "", + arguments: item.arguments || "{}", + }, + })); +} + /** * Parse safe JSON with fallback */