mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(bedrock): keep provider headers, tool search results, and encrypted reasoning on the Responses bridge
Bridged streams now carry the llm_provider-* response headers, a tool_reference tool result reaches Bedrock as its tool name like Converse sent it, and an extra_body include keeps reasoning.encrypted_content beside it. The Logs page Tools section now reads function calls from a Responses-shaped response, and the Converse tool search translation case pins the explicit converse route.
This commit is contained in:
parent
b66028f1a3
commit
222a9bf827
7 changed files with 173 additions and 23 deletions
|
|
@ -16,6 +16,8 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import httpx
|
||||
|
||||
from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse
|
||||
|
||||
|
||||
|
|
@ -32,6 +34,12 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
|
|||
client: ReadOnly[object]
|
||||
|
||||
|
||||
def _provider_response_headers(stream: object) -> "httpx.Headers | None":
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
|
||||
return stream.response.headers if isinstance(stream, BaseResponsesAPIStreamingIterator) else None
|
||||
|
||||
|
||||
def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
|
||||
"""`responses()` runs `get_llm_provider()` itself, so hand back the prefixed model `completion()` started from."""
|
||||
return f"{custom_llm_provider}/{model}"
|
||||
|
|
@ -269,6 +277,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
_response_headers=_provider_response_headers(result),
|
||||
)
|
||||
return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider)
|
||||
|
||||
|
|
@ -374,6 +383,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
_response_headers=_provider_response_headers(result),
|
||||
)
|
||||
return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider)
|
||||
|
||||
|
|
|
|||
|
|
@ -336,6 +336,28 @@ def reasoning_origin_for(provider: str | None, model: str) -> str | None:
|
|||
return f"{_REASONING_ORIGIN_PREFIX}bedrock/{get_bedrock_base_model(model)}:"
|
||||
|
||||
|
||||
def _tool_reference_as_text(part: object) -> object:
|
||||
match part:
|
||||
case {"type": "tool_reference", "tool_name": str() as tool_name}:
|
||||
return {"type": "text", "text": tool_name}
|
||||
case _:
|
||||
return part
|
||||
|
||||
|
||||
def _tool_message_with_tool_references_as_text(msg: Mapping[str, object]) -> Mapping[str, object]:
|
||||
content: Final = msg.get("content")
|
||||
if msg.get("role") != "tool" or not isinstance(content, list):
|
||||
return msg
|
||||
parts: Final = cast("list[object]", content) # cast-ok: isinstance leaves the part type unknown
|
||||
return {**msg, "content": [_tool_reference_as_text(part) for part in parts]}
|
||||
|
||||
|
||||
def _tool_references_as_text(messages: Sequence["AllMessageValues"]) -> tuple["AllMessageValues", ...]:
|
||||
return cast( # cast-ok: each message keeps its fields, and only tool content parts change
|
||||
'tuple["AllMessageValues", ...]', tuple(_tool_message_with_tool_references_as_text(msg) for msg in messages)
|
||||
)
|
||||
|
||||
|
||||
def _with_reasoning_origin(output_items: Sequence[object], origin: str | None) -> Sequence[object]:
|
||||
if origin is None:
|
||||
return output_items
|
||||
|
|
@ -859,7 +881,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
|
||||
def convert_chat_completion_messages_to_responses_api(
|
||||
self,
|
||||
messages: list["AllMessageValues"],
|
||||
messages: Sequence["AllMessageValues"],
|
||||
*,
|
||||
drop_params: bool = False,
|
||||
keep_prompt_cache_breakpoints: bool = False,
|
||||
|
|
@ -883,7 +905,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
|
||||
def _convert_chat_completion_messages_to_responses_input(
|
||||
self,
|
||||
messages: list["AllMessageValues"],
|
||||
messages: Sequence["AllMessageValues"],
|
||||
*,
|
||||
drop_params: bool = False,
|
||||
replay_tool_search: bool = False,
|
||||
|
|
@ -1140,7 +1162,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
and supports_openai_prompt_cache_breakpoint(base_model, provider)
|
||||
)
|
||||
converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
|
||||
messages,
|
||||
_tool_references_as_text(messages) if provider == "bedrock" else messages,
|
||||
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
|
||||
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
|
||||
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast
|
||||
|
|
|
|||
|
|
@ -1313,10 +1313,17 @@ def _bedrock_serves_responses(model: str) -> bool:
|
|||
def bedrock_responses_bridge_optional_params(optional_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
if optional_params.get("store") is not None:
|
||||
return optional_params
|
||||
include: Final = optional_params.get("include")
|
||||
extra_body: Final = optional_params.get("extra_body")
|
||||
extra_body_params: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body) if isinstance(extra_body, dict) else {}
|
||||
include: Final = extra_body_params.get("include", optional_params.get("include"))
|
||||
requested: Final = _INCLUDE_ADAPTER.validate_python(include) if isinstance(include, (list, tuple)) else ()
|
||||
return {
|
||||
**optional_params,
|
||||
**(
|
||||
{"extra_body": {key: value for key, value in extra_body_params.items() if key != "include"}}
|
||||
if "include" in extra_body_params
|
||||
else {}
|
||||
),
|
||||
"store": False,
|
||||
"include": [*requested, *(() if _ENCRYPTED_REASONING in requested else (_ENCRYPTED_REASONING,))],
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase(
|
|||
scenario="tool_search",
|
||||
litellm_endpoint="/v1/messages",
|
||||
litellm_request={
|
||||
"model": "bedrock/global.openai.gpt-6.1-sol",
|
||||
"model": "bedrock/converse/global.openai.gpt-6.1-sol",
|
||||
"max_tokens": 1024,
|
||||
"thinking": {"type": "adaptive"},
|
||||
"output_config": {"effort": "low"},
|
||||
|
|
@ -139,7 +139,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase(
|
|||
"id": ANY,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "bedrock/global.openai.gpt-6.1-sol",
|
||||
"model": "bedrock/converse/global.openai.gpt-6.1-sol",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 100, "output_tokens": 15},
|
||||
"content": [{"type": "text", "text": "The web search tool is loaded and ready to use."}],
|
||||
|
|
|
|||
|
|
@ -1238,6 +1238,80 @@ async def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(loca
|
|||
assert "include" not in body
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"include_param",
|
||||
[
|
||||
{"include": ["file_search_call.results"]},
|
||||
{"extra_body": {"include": ["file_search_call.results"]}},
|
||||
],
|
||||
)
|
||||
async def test_tools_with_reasoning_bridge_keeps_encrypted_reasoning_beside_a_requested_include(
|
||||
local_cost_map, fake_aws_env, include_param
|
||||
):
|
||||
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
|
||||
await litellm.acompletion(
|
||||
model="bedrock/global.openai.gpt-6-sol",
|
||||
messages=[{"role": "user", "content": "weather in Paris"}],
|
||||
tools=[GET_WEATHER_TOOL],
|
||||
reasoning_effort="low",
|
||||
client=client,
|
||||
**include_param,
|
||||
)
|
||||
|
||||
body = json.loads(requests[0].content)
|
||||
assert body["store"] is False
|
||||
assert body["include"] == ["file_search_call.results", "reasoning.encrypted_content"]
|
||||
|
||||
|
||||
async def test_tools_with_reasoning_bridge_sends_a_tool_reference_result_as_its_tool_name_like_converse(
|
||||
local_cost_map, fake_aws_env
|
||||
):
|
||||
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
|
||||
await litellm.acompletion(
|
||||
model="bedrock/global.openai.gpt-6-sol",
|
||||
messages=[
|
||||
{"role": "user", "content": "find a web tool"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "ToolSearch", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": [{"type": "tool_reference", "tool_name": "WebSearch"}],
|
||||
},
|
||||
],
|
||||
tools=[GET_WEATHER_TOOL],
|
||||
reasoning_effort="low",
|
||||
client=client,
|
||||
)
|
||||
|
||||
body = json.loads(requests[0].content)
|
||||
tool_output = next(item for item in body["input"] if item["type"] == "function_call_output")
|
||||
assert tool_output["output"] == [{"type": "input_text", "text": "WebSearch"}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["plain", "streamed"])
|
||||
async def test_tools_with_reasoning_bridge_returns_the_provider_response_headers(local_cost_map, fake_aws_env, stream):
|
||||
reply_kwargs: Final = (
|
||||
{
|
||||
"content": _responses_stream_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"),
|
||||
"headers": {"content-type": "text/event-stream", "x-amzn-requestid": "req-1"},
|
||||
}
|
||||
if stream
|
||||
else {
|
||||
"json": _responses_reply_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"),
|
||||
"headers": {"x-amzn-requestid": "req-1"},
|
||||
}
|
||||
)
|
||||
_, response = await _bedrock_turn([{"role": "user", "content": "weather in Paris"}], **reply_kwargs)
|
||||
|
||||
assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-1"
|
||||
|
||||
|
||||
async def test_tools_with_reasoning_and_a_param_the_bridge_drops_fail_like_converse_without_drop_params(
|
||||
local_cost_map, fake_aws_env
|
||||
):
|
||||
|
|
|
|||
|
|
@ -335,6 +335,38 @@ describe("ToolsSection utils", () => {
|
|||
expect(result[0].called).toBe(true);
|
||||
expect(result[0].callData?.arguments).toEqual({ location: "Tokyo" });
|
||||
});
|
||||
|
||||
it("should parse tool calls from a Responses API response output", () => {
|
||||
const log: Partial<LogEntry> = {
|
||||
request_id: "test-responses-1",
|
||||
proxy_server_request: {
|
||||
tools: [
|
||||
{ type: "function", function: { name: "get_weather", description: "Get current weather" } },
|
||||
{ type: "function", function: { name: "get_time", description: "Get current time" } },
|
||||
],
|
||||
},
|
||||
response: {
|
||||
object: "response",
|
||||
output: [
|
||||
{ type: "reasoning", summary: [] },
|
||||
{
|
||||
type: "function_call",
|
||||
call_id: "call_abc",
|
||||
name: "get_weather",
|
||||
arguments: '{"location": "Paris"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
} as any;
|
||||
|
||||
const result = parseToolsFromLog(log as LogEntry);
|
||||
|
||||
expect(result.map((tool) => [tool.name, tool.called])).toEqual([
|
||||
["get_weather", true],
|
||||
["get_time", false],
|
||||
]);
|
||||
expect(result[0].callData).toEqual({ id: "call_abc", name: "get_weather", arguments: { location: "Paris" } });
|
||||
});
|
||||
});
|
||||
|
||||
describe("hasTools", () => {
|
||||
|
|
|
|||
|
|
@ -80,31 +80,36 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] {
|
|||
return responseData.tool_calls;
|
||||
}
|
||||
|
||||
// Responses API format: response.output[].type === "function_call"
|
||||
if (Array.isArray(responseData.output)) {
|
||||
const toolCalls = toolCallsFromOutput(responseData.output);
|
||||
if (toolCalls.length > 0) return toolCalls;
|
||||
}
|
||||
|
||||
// Realtime API format: response.results[].response.output[].type === "function_call"
|
||||
if (Array.isArray(responseData.results)) {
|
||||
const toolCalls: ToolCall[] = [];
|
||||
for (const result of responseData.results) {
|
||||
if (result.type === "response.done" && result.response?.output) {
|
||||
for (const item of result.response.output) {
|
||||
if (item.type === "function_call") {
|
||||
toolCalls.push({
|
||||
id: item.call_id || "",
|
||||
type: "function",
|
||||
function: {
|
||||
name: item.name || "",
|
||||
arguments: item.arguments || "{}",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const toolCalls = responseData.results
|
||||
.filter((result: any) => result.type === "response.done" && Array.isArray(result.response?.output))
|
||||
.flatMap((result: any) => toolCallsFromOutput(result.response.output));
|
||||
if (toolCalls.length > 0) return toolCalls;
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
function toolCallsFromOutput(output: any[]): ToolCall[] {
|
||||
return output
|
||||
.filter((item: any) => item.type === "function_call")
|
||||
.map((item: any) => ({
|
||||
id: item.call_id || "",
|
||||
type: "function",
|
||||
function: {
|
||||
name: item.name || "",
|
||||
arguments: item.arguments || "{}",
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse safe JSON with fallback
|
||||
*/
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue