fix(bedrock): keep provider headers, tool search results, and encrypted reasoning on the Responses bridge

Bridged streams now carry the llm_provider-* response headers, a tool_reference
tool result reaches Bedrock as its tool name like Converse sent it, and an
extra_body include keeps reasoning.encrypted_content beside it. The Logs page
Tools section now reads function calls from a Responses-shaped response, and the
Converse tool search translation case pins the explicit converse route.
This commit is contained in:
mateo-berri 2026-10-20 23:34:01 -07:00
parent b66028f1a3
commit 222a9bf827
7 changed files with 173 additions and 23 deletions

View file

@ -16,6 +16,8 @@ from typing_extensions import ReadOnly, TypedDict
from litellm.types.llms.openai import ResponsesAPIResponse
if TYPE_CHECKING:
import httpx
from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse
@ -32,6 +34,12 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
client: ReadOnly[object]
def _provider_response_headers(stream: object) -> "httpx.Headers | None":
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
return stream.response.headers if isinstance(stream, BaseResponsesAPIStreamingIterator) else None
def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
"""`responses()` runs `get_llm_provider()` itself, so hand back the prefixed model `completion()` started from."""
return f"{custom_llm_provider}/{model}"
@ -269,6 +277,7 @@ class ResponsesToCompletionBridgeHandler:
model=model,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
_response_headers=_provider_response_headers(result),
)
return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider)
@ -374,6 +383,7 @@ class ResponsesToCompletionBridgeHandler:
model=model,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
_response_headers=_provider_response_headers(result),
)
return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider)

View file

@ -336,6 +336,28 @@ def reasoning_origin_for(provider: str | None, model: str) -> str | None:
return f"{_REASONING_ORIGIN_PREFIX}bedrock/{get_bedrock_base_model(model)}:"
def _tool_reference_as_text(part: object) -> object:
match part:
case {"type": "tool_reference", "tool_name": str() as tool_name}:
return {"type": "text", "text": tool_name}
case _:
return part
def _tool_message_with_tool_references_as_text(msg: Mapping[str, object]) -> Mapping[str, object]:
content: Final = msg.get("content")
if msg.get("role") != "tool" or not isinstance(content, list):
return msg
parts: Final = cast("list[object]", content) # cast-ok: isinstance leaves the part type unknown
return {**msg, "content": [_tool_reference_as_text(part) for part in parts]}
def _tool_references_as_text(messages: Sequence["AllMessageValues"]) -> tuple["AllMessageValues", ...]:
return cast( # cast-ok: each message keeps its fields, and only tool content parts change
'tuple["AllMessageValues", ...]', tuple(_tool_message_with_tool_references_as_text(msg) for msg in messages)
)
def _with_reasoning_origin(output_items: Sequence[object], origin: str | None) -> Sequence[object]:
if origin is None:
return output_items
@ -859,7 +881,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def convert_chat_completion_messages_to_responses_api(
self,
messages: list["AllMessageValues"],
messages: Sequence["AllMessageValues"],
*,
drop_params: bool = False,
keep_prompt_cache_breakpoints: bool = False,
@ -883,7 +905,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def _convert_chat_completion_messages_to_responses_input(
self,
messages: list["AllMessageValues"],
messages: Sequence["AllMessageValues"],
*,
drop_params: bool = False,
replay_tool_search: bool = False,
@ -1140,7 +1162,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
and supports_openai_prompt_cache_breakpoint(base_model, provider)
)
converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
messages,
_tool_references_as_text(messages) if provider == "bedrock" else messages,
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast

View file

@ -1313,10 +1313,17 @@ def _bedrock_serves_responses(model: str) -> bool:
def bedrock_responses_bridge_optional_params(optional_params: Mapping[str, object]) -> Mapping[str, object]:
if optional_params.get("store") is not None:
return optional_params
include: Final = optional_params.get("include")
extra_body: Final = optional_params.get("extra_body")
extra_body_params: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body) if isinstance(extra_body, dict) else {}
include: Final = extra_body_params.get("include", optional_params.get("include"))
requested: Final = _INCLUDE_ADAPTER.validate_python(include) if isinstance(include, (list, tuple)) else ()
return {
**optional_params,
**(
{"extra_body": {key: value for key, value in extra_body_params.items() if key != "include"}}
if "include" in extra_body_params
else {}
),
"store": False,
"include": [*requested, *(() if _ENCRYPTED_REASONING in requested else (_ENCRYPTED_REASONING,))],
}

View file

@ -11,7 +11,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase(
scenario="tool_search",
litellm_endpoint="/v1/messages",
litellm_request={
"model": "bedrock/global.openai.gpt-6.1-sol",
"model": "bedrock/converse/global.openai.gpt-6.1-sol",
"max_tokens": 1024,
"thinking": {"type": "adaptive"},
"output_config": {"effort": "low"},
@ -139,7 +139,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase(
"id": ANY,
"type": "message",
"role": "assistant",
"model": "bedrock/global.openai.gpt-6.1-sol",
"model": "bedrock/converse/global.openai.gpt-6.1-sol",
"stop_sequence": None,
"usage": {"input_tokens": 100, "output_tokens": 15},
"content": [{"type": "text", "text": "The web search tool is loaded and ready to use."}],

View file

@ -1238,6 +1238,80 @@ async def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(loca
assert "include" not in body
@pytest.mark.parametrize(
"include_param",
[
{"include": ["file_search_call.results"]},
{"extra_body": {"include": ["file_search_call.results"]}},
],
)
async def test_tools_with_reasoning_bridge_keeps_encrypted_reasoning_beside_a_requested_include(
local_cost_map, fake_aws_env, include_param
):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
client=client,
**include_param,
)
body = json.loads(requests[0].content)
assert body["store"] is False
assert body["include"] == ["file_search_call.results", "reasoning.encrypted_content"]
async def test_tools_with_reasoning_bridge_sends_a_tool_reference_result_as_its_tool_name_like_converse(
local_cost_map, fake_aws_env
):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[
{"role": "user", "content": "find a web tool"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "ToolSearch", "arguments": "{}"}}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": [{"type": "tool_reference", "tool_name": "WebSearch"}],
},
],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
client=client,
)
body = json.loads(requests[0].content)
tool_output = next(item for item in body["input"] if item["type"] == "function_call_output")
assert tool_output["output"] == [{"type": "input_text", "text": "WebSearch"}]
@pytest.mark.parametrize("stream", [False, True], ids=["plain", "streamed"])
async def test_tools_with_reasoning_bridge_returns_the_provider_response_headers(local_cost_map, fake_aws_env, stream):
reply_kwargs: Final = (
{
"content": _responses_stream_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"),
"headers": {"content-type": "text/event-stream", "x-amzn-requestid": "req-1"},
}
if stream
else {
"json": _responses_reply_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"),
"headers": {"x-amzn-requestid": "req-1"},
}
)
_, response = await _bedrock_turn([{"role": "user", "content": "weather in Paris"}], **reply_kwargs)
assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-1"
async def test_tools_with_reasoning_and_a_param_the_bridge_drops_fail_like_converse_without_drop_params(
local_cost_map, fake_aws_env
):

View file

@ -335,6 +335,38 @@ describe("ToolsSection utils", () => {
expect(result[0].called).toBe(true);
expect(result[0].callData?.arguments).toEqual({ location: "Tokyo" });
});
it("should parse tool calls from a Responses API response output", () => {
const log: Partial<LogEntry> = {
request_id: "test-responses-1",
proxy_server_request: {
tools: [
{ type: "function", function: { name: "get_weather", description: "Get current weather" } },
{ type: "function", function: { name: "get_time", description: "Get current time" } },
],
},
response: {
object: "response",
output: [
{ type: "reasoning", summary: [] },
{
type: "function_call",
call_id: "call_abc",
name: "get_weather",
arguments: '{"location": "Paris"}',
},
],
},
} as any;
const result = parseToolsFromLog(log as LogEntry);
expect(result.map((tool) => [tool.name, tool.called])).toEqual([
["get_weather", true],
["get_time", false],
]);
expect(result[0].callData).toEqual({ id: "call_abc", name: "get_weather", arguments: { location: "Paris" } });
});
});
describe("hasTools", () => {

View file

@ -80,31 +80,36 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] {
return responseData.tool_calls;
}
// Responses API format: response.output[].type === "function_call"
if (Array.isArray(responseData.output)) {
const toolCalls = toolCallsFromOutput(responseData.output);
if (toolCalls.length > 0) return toolCalls;
}
// Realtime API format: response.results[].response.output[].type === "function_call"
if (Array.isArray(responseData.results)) {
const toolCalls: ToolCall[] = [];
for (const result of responseData.results) {
if (result.type === "response.done" && result.response?.output) {
for (const item of result.response.output) {
if (item.type === "function_call") {
toolCalls.push({
id: item.call_id || "",
type: "function",
function: {
name: item.name || "",
arguments: item.arguments || "{}",
},
});
}
}
}
}
const toolCalls = responseData.results
.filter((result: any) => result.type === "response.done" && Array.isArray(result.response?.output))
.flatMap((result: any) => toolCallsFromOutput(result.response.output));
if (toolCalls.length > 0) return toolCalls;
}
return [];
}
function toolCallsFromOutput(output: any[]): ToolCall[] {
return output
.filter((item: any) => item.type === "function_call")
.map((item: any) => ({
id: item.call_id || "",
type: "function",
function: {
name: item.name || "",
arguments: item.arguments || "{}",
},
}));
}
/**
* Parse safe JSON with fallback
*/