mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-22 00:31:44 +00:00
feat: honor eager_input_streaming on Bedrock and Anthropic Claude tools
This commit is contained in:
mateo-berri
parent
a2626726a2
commit
d1563e0b55
16 changed files with 444 additions and 13 deletions
|
|
@ -94,6 +94,7 @@ from litellm.utils import (
|
|||
from ..common_utils import (
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
eager_input_streaming_flag,
|
||||
process_anthropic_headers,
|
||||
strip_advisor_blocks_from_messages,
|
||||
)
|
||||
|
|
@ -732,10 +733,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
|
||||
input_anthropic_schema: Final = sanitize_input_schema_for_anthropic(_input_schema)
|
||||
|
||||
_tool: Final = AnthropicMessagesTool(
|
||||
name=tool["function"]["name"],
|
||||
input_schema=input_anthropic_schema,
|
||||
type="custom",
|
||||
_eager_input_streaming: Final = eager_input_streaming_flag(tool)
|
||||
_tool: Final = (
|
||||
AnthropicMessagesTool(
|
||||
name=tool["function"]["name"],
|
||||
input_schema=input_anthropic_schema,
|
||||
type="custom",
|
||||
)
|
||||
if _eager_input_streaming is None
|
||||
else AnthropicMessagesTool(
|
||||
name=tool["function"]["name"],
|
||||
input_schema=input_anthropic_schema,
|
||||
type="custom",
|
||||
eager_input_streaming=_eager_input_streaming,
|
||||
)
|
||||
)
|
||||
|
||||
_description: Final = tool["function"].get("description")
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from types import MappingProxyType
|
|||
from typing import Any, Final, Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
|
|
@ -19,6 +19,7 @@ from litellm.constants import (
|
|||
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
|
||||
)
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_file_ids_from_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
|
|
@ -231,6 +232,27 @@ def optionally_handle_anthropic_oauth(headers: dict, api_key: str | None) -> tup
|
|||
return headers, api_key
|
||||
|
||||
|
||||
class _EagerInputStreamingFunction(BaseModel):
|
||||
eager_input_streaming: StrictBool | None = None
|
||||
|
||||
|
||||
class _EagerInputStreamingTool(BaseModel):
|
||||
eager_input_streaming: StrictBool | None = None
|
||||
function: _EagerInputStreamingFunction | None = None
|
||||
|
||||
|
||||
def eager_input_streaming_flag(tool: object) -> bool | None:
|
||||
try:
|
||||
parsed: Final = _EagerInputStreamingTool.model_validate(tool)
|
||||
except ValidationError as error:
|
||||
if isinstance(tool, Mapping):
|
||||
raise UnsupportedParamsError(message="eager_input_streaming must be a boolean") from error
|
||||
return None
|
||||
if parsed.eager_input_streaming is not None:
|
||||
return parsed.eager_input_streaming
|
||||
return parsed.function.eager_input_streaming if parsed.function is not None else None
|
||||
|
||||
|
||||
class AnthropicError(BaseLLMException):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -373,6 +395,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
|
||||
return False
|
||||
|
||||
def is_eager_input_streaming_used(self, tools: Sequence[object] | None) -> bool:
|
||||
return any(eager_input_streaming_flag(tool) is True for tool in tools or ())
|
||||
|
||||
@staticmethod
|
||||
def _supports_sampling_params(model: str) -> bool:
|
||||
"""Claude 4.7+ (Opus 4.7/4.8, Fable 5) removed sampling params: the API
|
||||
|
|
|
|||
|
|
@ -111,6 +111,7 @@ from litellm.litellm_core_utils.reasoning_effort_utils import (
|
|||
reasoning_effort_from_thinking_budget,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
eager_input_streaming_flag,
|
||||
is_empty_unsigned_thinking_block,
|
||||
normalize_anthropic_tool_use_id,
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
|
|
@ -770,6 +771,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
"cache_control",
|
||||
"strict",
|
||||
"type",
|
||||
"eager_input_streaming",
|
||||
]
|
||||
|
||||
for idx, tool in enumerate(tools):
|
||||
|
|
@ -808,7 +810,14 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
for k, v in tool.items():
|
||||
if k not in mapped_tool_params: # pass additional computer kwargs
|
||||
function_chunk.setdefault("parameters", {}).update({k: v})
|
||||
tool_param = ChatCompletionToolParam(type="function", function=function_chunk)
|
||||
eager_input_streaming = eager_input_streaming_flag(tool)
|
||||
tool_param = (
|
||||
ChatCompletionToolParam(type="function", function=function_chunk)
|
||||
if eager_input_streaming is None
|
||||
else ChatCompletionToolParam(
|
||||
type="function", function=function_chunk, eager_input_streaming=eager_input_streaming
|
||||
)
|
||||
)
|
||||
self._add_cache_control_if_applicable(tool, tool_param, model)
|
||||
new_tools.append(tool_param)
|
||||
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ from litellm.llms.bedrock.request_metadata import (
|
|||
merge_bedrock_invoke_headers,
|
||||
resolve_bedrock_request_metadata,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER
|
||||
from litellm.types.llms.bedrock import *
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -1518,10 +1519,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
bedrock_tools: list[ToolBlock] = []
|
||||
|
||||
# Collect anthropic_beta values from user headers
|
||||
anthropic_beta_list: Final = []
|
||||
if headers:
|
||||
user_betas: Final = get_anthropic_beta_from_headers(headers)
|
||||
anthropic_beta_list.extend(user_betas)
|
||||
anthropic_beta_list: Final = list(get_anthropic_beta_from_headers(headers or {}))
|
||||
|
||||
# Separate pre-formatted Bedrock tools (e.g. systemTool from web_search_options)
|
||||
# from OpenAI-format tools that need transformation via _bedrock_tools_pt
|
||||
|
|
@ -1634,6 +1632,12 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if ANTHROPIC_EFFORT_BETA_HEADER not in anthropic_beta_list:
|
||||
anthropic_beta_list.append(ANTHROPIC_EFFORT_BETA_HEADER)
|
||||
|
||||
if (
|
||||
AnthropicModelInfo().is_eager_input_streaming_used(filtered_tools)
|
||||
and ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER not in anthropic_beta_list
|
||||
):
|
||||
anthropic_beta_list.append(ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER)
|
||||
|
||||
# Bedrock Converse: compact_20260112 edits only (+ beta header).
|
||||
AmazonConverseConfig._filter_context_management_for_bedrock_converse(
|
||||
additional_request_params, anthropic_beta_list
|
||||
|
|
|
|||
|
|
@ -24,8 +24,12 @@ from litellm.llms.bedrock.common_utils import (
|
|||
normalize_custom_field_on_tools,
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke,
|
||||
strip_unsupported_bedrock_invoke_output_config_keys,
|
||||
tools_without_eager_input_streaming,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,
|
||||
ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
|
@ -237,6 +241,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
# Hoist `custom.defer_loading` then drop `custom` (Bedrock doesn't support it)
|
||||
normalize_custom_field_on_tools(anthropic_request)
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke(anthropic_request)
|
||||
outbound_tools: Final = tools_without_eager_input_streaming(anthropic_request)
|
||||
if outbound_tools is not None:
|
||||
anthropic_request["tools"] = outbound_tools
|
||||
return anthropic_request
|
||||
|
||||
def _compute_bedrock_invoke_beta_headers(
|
||||
|
|
@ -269,6 +276,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
if bedrock_supports_tool_search(model):
|
||||
beta_set.add("tool-search-tool-2025-10-19")
|
||||
|
||||
if self.is_eager_input_streaming_used(tools):
|
||||
beta_set.add(ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER)
|
||||
|
||||
auto_beta_list: Final = filter_and_transform_beta_headers(
|
||||
beta_headers=list(beta_set - user_beta_set),
|
||||
provider="bedrock",
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import functools
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -18,6 +18,7 @@ if TYPE_CHECKING:
|
|||
from litellm.types.llms.bedrock import BedrockCreateBatchRequest
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -330,6 +331,17 @@ def normalize_custom_field_on_tools(request_body: dict) -> None:
|
|||
tool["defer_loading"] = deferred
|
||||
|
||||
|
||||
_TOOL_DICTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
def tools_without_eager_input_streaming(request_body: Mapping[str, object]) -> Sequence[object] | None:
|
||||
try:
|
||||
tools: Final = _TOOL_DICTS_ADAPTER.validate_python(request_body.get("tools"))
|
||||
except ValidationError:
|
||||
return None
|
||||
return [{key: value for key, value in tool.items() if key != "eager_input_streaming"} for tool in tools]
|
||||
|
||||
|
||||
def normalize_json_schema_custom_types_to_object(schema: dict) -> None:
|
||||
"""
|
||||
In-place: replace JSON Schema ``type: \"custom\"`` with ``\"object\"`` (iterative walk).
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
normalize_custom_field_on_tools,
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke,
|
||||
strip_unsupported_bedrock_invoke_output_config_keys,
|
||||
tools_without_eager_input_streaming,
|
||||
)
|
||||
from litellm.llms.bedrock.request_metadata import (
|
||||
bedrock_request_metadata_headers,
|
||||
|
|
@ -46,6 +47,7 @@ from litellm.llms.bedrock.request_metadata import (
|
|||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,
|
||||
ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
|
||||
)
|
||||
from litellm.types.llms.bedrock import BedrockInvokeAnthropicMessagesRequest
|
||||
|
|
@ -525,6 +527,9 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
if injected_thinking_for_clear_thinking:
|
||||
beta_set.add("interleaved-thinking-2025-05-14")
|
||||
|
||||
if anthropic_model_info.is_eager_input_streaming_used(tools):
|
||||
beta_set.add(ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER)
|
||||
|
||||
self._filter_context_management_for_bedrock_invoke(
|
||||
anthropic_messages_request=anthropic_messages_request,
|
||||
beta_set=beta_set,
|
||||
|
|
@ -719,6 +724,10 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
if filtered_betas:
|
||||
anthropic_messages_request["anthropic_beta"] = filtered_betas
|
||||
|
||||
outbound_tools: Final = tools_without_eager_input_streaming(anthropic_messages_request)
|
||||
if outbound_tools is not None:
|
||||
anthropic_messages_request["tools"] = outbound_tools
|
||||
|
||||
remaining_output_config: Final = anthropic_messages_request.get("output_config")
|
||||
if (
|
||||
litellm.drop_params is True
|
||||
|
|
|
|||
|
|
@ -19370,7 +19370,7 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"description": "\n Unified rate-limit error.\n\n Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n The :attr:`category` attribute lets callers distinguish the source. See\n :class:`RateLimitErrorCategory` for the available values.\n "
|
||||
"description": "\nUnified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values.\n"
|
||||
},
|
||||
"500": {
|
||||
"content": {
|
||||
|
|
@ -32899,6 +32899,10 @@
|
|||
"cache_control": {
|
||||
"$ref": "#/components/schemas/ChatCompletionCachedContent"
|
||||
},
|
||||
"eager_input_streaming": {
|
||||
"title": "Eager Input Streaming",
|
||||
"type": "boolean"
|
||||
},
|
||||
"function": {
|
||||
"$ref": "#/components/schemas/ChatCompletionToolParamFunctionChunk"
|
||||
},
|
||||
|
|
@ -32928,6 +32932,10 @@
|
|||
"title": "Description",
|
||||
"type": "string"
|
||||
},
|
||||
"eager_input_streaming": {
|
||||
"title": "Eager Input Streaming",
|
||||
"type": "boolean"
|
||||
},
|
||||
"name": {
|
||||
"title": "Name",
|
||||
"type": "string"
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ class AnthropicMessagesTool(TypedDict, total=False):
|
|||
defer_loading: bool
|
||||
allowed_callers: list[str] | None
|
||||
input_examples: list[dict[str, Any]] | None
|
||||
eager_input_streaming: ReadOnly[bool]
|
||||
|
||||
|
||||
class AnthropicComputerTool(TypedDict, total=False):
|
||||
|
|
@ -755,6 +756,8 @@ ANTHROPIC_TOOL_SEARCH_BETA_HEADER: Final = "advanced-tool-use-2025-11-20"
|
|||
# Effort beta header constant
|
||||
ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24"
|
||||
|
||||
ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER: Final = "fine-grained-tool-streaming-2025-05-14"
|
||||
|
||||
# OAuth constants
|
||||
ANTHROPIC_OAUTH_TOKEN_PREFIX: Final = "sk-ant-oat"
|
||||
ANTHROPIC_OAUTH_BETA_HEADER: Final = "oauth-2025-04-20"
|
||||
|
|
|
|||
|
|
@ -992,6 +992,7 @@ class ChatCompletionToolParamFunctionChunk(TypedDict, total=False):
|
|||
description: str
|
||||
parameters: dict
|
||||
strict: bool
|
||||
eager_input_streaming: ReadOnly[bool]
|
||||
|
||||
|
||||
class OpenAIChatCompletionToolParam(TypedDict):
|
||||
|
|
@ -1002,6 +1003,7 @@ class OpenAIChatCompletionToolParam(TypedDict):
|
|||
class ChatCompletionToolParam(OpenAIChatCompletionToolParam, total=False):
|
||||
cache_control: ChatCompletionCachedContent
|
||||
allowed_callers: list[str]
|
||||
eager_input_streaming: ReadOnly[bool]
|
||||
|
||||
|
||||
class Function(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -6370,3 +6370,74 @@ def test_response_format_tool_path_skips_forced_tool_choice_when_unsupported(loc
|
|||
|
||||
assert "tools" in result
|
||||
assert "tool_choice" not in result
|
||||
|
||||
|
||||
def _eager_chat_tool(**extra):
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "write_file",
|
||||
"description": "Write a file",
|
||||
"parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]},
|
||||
},
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", [True, False])
|
||||
def test_eager_input_streaming_passed_through_from_tool_top_level(flag):
|
||||
mapped_tool, _ = AnthropicConfig()._map_tool_helper(_eager_chat_tool(eager_input_streaming=flag))
|
||||
|
||||
assert mapped_tool == {
|
||||
"name": "write_file",
|
||||
"description": "Write a file",
|
||||
"input_schema": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]},
|
||||
"type": "custom",
|
||||
"eager_input_streaming": flag,
|
||||
}
|
||||
|
||||
|
||||
def test_eager_input_streaming_passed_through_from_function():
|
||||
tool = _eager_chat_tool()
|
||||
tool["function"]["eager_input_streaming"] = True
|
||||
|
||||
mapped_tool, _ = AnthropicConfig()._map_tool_helper(tool)
|
||||
|
||||
assert mapped_tool["eager_input_streaming"] is True
|
||||
assert "eager_input_streaming" not in mapped_tool["input_schema"]
|
||||
|
||||
|
||||
def test_eager_input_streaming_absent_stays_absent():
|
||||
mapped_tool, _ = AnthropicConfig()._map_tool_helper(_eager_chat_tool())
|
||||
|
||||
assert "eager_input_streaming" not in mapped_tool
|
||||
|
||||
|
||||
def test_eager_input_streaming_rejects_non_boolean():
|
||||
with pytest.raises(litellm.BadRequestError, match="eager_input_streaming must be a boolean"):
|
||||
AnthropicConfig()._map_tool_helper(_eager_chat_tool(eager_input_streaming="true"))
|
||||
|
||||
|
||||
def test_eager_input_streaming_not_set_on_computer_use_tool():
|
||||
computer_tool = {
|
||||
"type": "computer_20250124",
|
||||
"function": {"name": "computer", "parameters": {"display_width_px": 1024, "display_height_px": 768}},
|
||||
"eager_input_streaming": True,
|
||||
}
|
||||
|
||||
mapped_tool, _ = AnthropicConfig()._map_tool_helper(computer_tool)
|
||||
|
||||
assert mapped_tool["type"] == "computer_20250124"
|
||||
assert "eager_input_streaming" not in mapped_tool
|
||||
|
||||
|
||||
def test_eager_input_streaming_reaches_anthropic_request_tools():
|
||||
result = AnthropicConfig().map_openai_params(
|
||||
non_default_params={"tools": [_eager_chat_tool(eager_input_streaming=True)], "stream": True},
|
||||
optional_params={},
|
||||
model="claude-sonnet-5",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["tools"][0]["eager_input_streaming"] is True
|
||||
assert result["tools"][0]["name"] == "write_file"
|
||||
|
|
|
|||
|
|
@ -5017,3 +5017,45 @@ def test_redacted_thinking_blocks_never_carry_cache_control():
|
|||
replayed: Final = outbound["messages"][1]["content"][0]
|
||||
assert replayed["type"] == "redacted_thinking"
|
||||
assert "cache_control" not in replayed
|
||||
|
||||
|
||||
EAGER_INPUT_SCHEMA: Final = {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", [True, False])
|
||||
def test_translate_anthropic_tools_to_openai_carries_eager_input_streaming_onto_tool(flag):
|
||||
"""The per-tool flag lands on the OpenAI tool object, never inside the JSON schema Bedrock sends as inputSchema."""
|
||||
tools: Final = [{"name": "write_file", "input_schema": EAGER_INPUT_SCHEMA, "eager_input_streaming": flag}]
|
||||
|
||||
new_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(tools=tools)
|
||||
|
||||
assert new_tools[0]["eager_input_streaming"] is flag
|
||||
assert new_tools[0]["function"]["parameters"] == EAGER_INPUT_SCHEMA
|
||||
assert "eager_input_streaming" not in new_tools[0]["function"]
|
||||
|
||||
|
||||
def test_translate_anthropic_tools_to_openai_omits_unset_eager_input_streaming():
|
||||
tools: Final = [{"name": "write_file", "input_schema": EAGER_INPUT_SCHEMA}]
|
||||
|
||||
new_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(tools=tools)
|
||||
|
||||
assert "eager_input_streaming" not in new_tools[0]
|
||||
assert "eager_input_streaming" not in new_tools[0]["function"]["parameters"]
|
||||
|
||||
|
||||
def test_eager_input_streaming_tool_reaches_bedrock_converse_as_beta():
|
||||
"""An Anthropic Messages request routed to bedrock/converse/ turns the flag into the fine-grained streaming beta."""
|
||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||
|
||||
tools: Final = [{"name": "write_file", "input_schema": EAGER_INPUT_SCHEMA, "eager_input_streaming": True}]
|
||||
new_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(tools=tools)
|
||||
|
||||
data: Final = AmazonConverseConfig()._transform_request_helper(
|
||||
model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
system_content_blocks=[],
|
||||
optional_params={"tools": new_tools},
|
||||
messages=[{"role": "user", "content": "write a big file"}],
|
||||
)
|
||||
|
||||
assert data["additionalModelRequestFields"]["anthropic_beta"] == ["fine-grained-tool-streaming-2025-05-14"]
|
||||
assert data["toolConfig"]["tools"][0]["toolSpec"]["inputSchema"]["json"] == EAGER_INPUT_SCHEMA
|
||||
|
|
|
|||
|
|
@ -859,3 +859,56 @@ def test_bedrock_chat_invoke_tool_search_beta_follows_model_map(
|
|||
)
|
||||
|
||||
assert result.get("anthropic_beta") == expected_betas
|
||||
|
||||
|
||||
FINE_GRAINED_TOOL_STREAMING_BETA = "fine-grained-tool-streaming-2025-05-14"
|
||||
EAGER_TOOL_SCHEMA = {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}
|
||||
|
||||
|
||||
def _chat_invoke_request_with_tools(tools, headers=None):
|
||||
config = AmazonAnthropicClaudeConfig()
|
||||
model = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
optional_params = config.map_openai_params(
|
||||
non_default_params={"max_tokens": 64, "stream": True, "tools": tools},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
return config.transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "write a big file"}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers=headers or {},
|
||||
)
|
||||
|
||||
|
||||
def _eager_openai_tool(name, **extra):
|
||||
return {"type": "function", "function": {"name": name, "parameters": EAGER_TOOL_SCHEMA}, **extra}
|
||||
|
||||
|
||||
def test_bedrock_chat_invoke_eager_input_streaming_tool_adds_beta_and_strips_key():
|
||||
result = _chat_invoke_request_with_tools(
|
||||
[_eager_openai_tool("write_file", eager_input_streaming=True), _eager_openai_tool("read_file")]
|
||||
)
|
||||
|
||||
assert result["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA]
|
||||
assert [tool["name"] for tool in result["tools"]] == ["write_file", "read_file"]
|
||||
assert all("eager_input_streaming" not in tool for tool in result["tools"])
|
||||
assert result["tools"][0]["input_schema"] == EAGER_TOOL_SCHEMA
|
||||
|
||||
|
||||
def test_bedrock_chat_invoke_eager_input_streaming_false_strips_key_without_beta():
|
||||
result = _chat_invoke_request_with_tools([_eager_openai_tool("write_file", eager_input_streaming=False)])
|
||||
|
||||
assert "anthropic_beta" not in result
|
||||
assert "eager_input_streaming" not in result["tools"][0]
|
||||
|
||||
|
||||
def test_bedrock_chat_invoke_eager_input_streaming_beta_not_duplicated_with_client_header():
|
||||
result = _chat_invoke_request_with_tools(
|
||||
[_eager_openai_tool("write_file", eager_input_streaming=True)],
|
||||
headers={"anthropic-beta": FINE_GRAINED_TOOL_STREAMING_BETA},
|
||||
)
|
||||
|
||||
assert result["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA]
|
||||
|
|
|
|||
|
|
@ -7400,3 +7400,109 @@ def test_transform_response_honors_json_mode_kwarg_when_optional_params_lack_it(
|
|||
)
|
||||
assert result.choices[0].message.tool_calls is None
|
||||
assert json.loads(result.choices[0].message.content) == {"city": "Paris", "population": 2100000}
|
||||
|
||||
|
||||
FINE_GRAINED_TOOL_STREAMING_BETA = "fine-grained-tool-streaming-2025-05-14"
|
||||
EAGER_TOOL_SCHEMA = {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}
|
||||
|
||||
|
||||
def _eager_openai_tool(**extra):
|
||||
return {"type": "function", "function": {"name": "write_file", "parameters": EAGER_TOOL_SCHEMA}, **extra}
|
||||
|
||||
|
||||
def _eager_openai_function_tool(**extra):
|
||||
return {"type": "function", "function": {"name": "write_file", "parameters": EAGER_TOOL_SCHEMA, **extra}}
|
||||
|
||||
|
||||
def _eager_anthropic_tool(**extra):
|
||||
return {"name": "write_file", "input_schema": EAGER_TOOL_SCHEMA, **extra}
|
||||
|
||||
|
||||
def _converse_request(model, tools, headers=None):
|
||||
return AmazonConverseConfig()._transform_request_helper(
|
||||
model=model,
|
||||
system_content_blocks=[],
|
||||
optional_params={"tools": tools},
|
||||
messages=[{"role": "user", "content": "write a big file"}],
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
[
|
||||
_eager_openai_tool(eager_input_streaming=True),
|
||||
_eager_openai_function_tool(eager_input_streaming=True),
|
||||
_eager_anthropic_tool(eager_input_streaming=True),
|
||||
],
|
||||
ids=["openai_top_level", "openai_under_function", "anthropic_shape"],
|
||||
)
|
||||
def test_eager_input_streaming_tool_adds_fine_grained_tool_streaming_beta(tool):
|
||||
data = _converse_request("us.anthropic.claude-sonnet-4-5-20250929-v1:0", [tool])
|
||||
|
||||
assert data["additionalModelRequestFields"]["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA]
|
||||
tool_spec = data["toolConfig"]["tools"][0]["toolSpec"]
|
||||
assert tool_spec["name"] == "write_file"
|
||||
assert "eager_input_streaming" not in tool_spec
|
||||
assert "eager_input_streaming" not in tool_spec["inputSchema"]["json"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
[
|
||||
_eager_openai_tool(eager_input_streaming=False),
|
||||
_eager_openai_function_tool(eager_input_streaming=False),
|
||||
_eager_anthropic_tool(eager_input_streaming=False),
|
||||
_eager_openai_tool(),
|
||||
],
|
||||
ids=["openai_false", "function_false", "anthropic_false", "absent"],
|
||||
)
|
||||
def test_eager_input_streaming_false_or_absent_adds_no_beta(tool):
|
||||
data = _converse_request("us.anthropic.claude-sonnet-4-5-20250929-v1:0", [tool])
|
||||
|
||||
assert "anthropic_beta" not in data.get("additionalModelRequestFields", {})
|
||||
assert "eager_input_streaming" not in data["toolConfig"]["tools"][0]["toolSpec"]
|
||||
|
||||
|
||||
def test_eager_input_streaming_beta_only_on_anthropic_models():
|
||||
data = _converse_request("amazon.nova-pro-v1:0", [_eager_openai_tool(eager_input_streaming=True)])
|
||||
|
||||
assert "anthropic_beta" not in data.get("additionalModelRequestFields", {})
|
||||
assert data["toolConfig"]["tools"][0]["toolSpec"]["name"] == "write_file"
|
||||
|
||||
|
||||
def test_eager_input_streaming_beta_not_duplicated_with_client_header():
|
||||
data = _converse_request(
|
||||
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
[_eager_openai_tool(eager_input_streaming=True)],
|
||||
headers={"anthropic-beta": f"{FINE_GRAINED_TOOL_STREAMING_BETA},interleaved-thinking-2025-05-14"},
|
||||
)
|
||||
|
||||
assert data["additionalModelRequestFields"]["anthropic_beta"] == [
|
||||
FINE_GRAINED_TOOL_STREAMING_BETA,
|
||||
"interleaved-thinking-2025-05-14",
|
||||
]
|
||||
|
||||
|
||||
def test_eager_input_streaming_beta_never_written_back_into_client_header_list():
|
||||
headers = {"anthropic-beta": ["interleaved-thinking-2025-05-14"]}
|
||||
|
||||
data = _converse_request(
|
||||
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
[_eager_openai_tool(eager_input_streaming=True)],
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
assert data["additionalModelRequestFields"]["anthropic_beta"] == [
|
||||
"interleaved-thinking-2025-05-14",
|
||||
FINE_GRAINED_TOOL_STREAMING_BETA,
|
||||
]
|
||||
assert headers == {"anthropic-beta": ["interleaved-thinking-2025-05-14"]}
|
||||
|
||||
|
||||
def test_eager_input_streaming_non_boolean_is_a_bad_request():
|
||||
with pytest.raises(litellm.BadRequestError, match="eager_input_streaming must be a boolean"):
|
||||
_converse_request(
|
||||
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
[_eager_openai_tool(eager_input_streaming="true")],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3244,3 +3244,65 @@ def test_bedrock_messages_strips_effort_but_keeps_format_for_sonnet_4_5(local_mo
|
|||
)
|
||||
|
||||
assert result.get("output_config") == {"format": schema_format}
|
||||
|
||||
|
||||
FINE_GRAINED_TOOL_STREAMING_BETA = "fine-grained-tool-streaming-2025-05-14"
|
||||
|
||||
|
||||
def _invoke_request_with_tools(tools, headers=None):
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
return AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
|
||||
model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
messages=[{"role": "user", "content": "write a big file"}],
|
||||
anthropic_messages_optional_request_params={"max_tokens": 4096, "tools": copy.deepcopy(tools), "stream": True},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers=headers or {},
|
||||
)
|
||||
|
||||
|
||||
def _eager_invoke_tool(name, eager_input_streaming):
|
||||
return {
|
||||
"name": name,
|
||||
"description": f"{name} tool",
|
||||
"input_schema": {"type": "object", "properties": {"path": {"type": "string"}}},
|
||||
"eager_input_streaming": eager_input_streaming,
|
||||
}
|
||||
|
||||
|
||||
def test_bedrock_invoke_eager_input_streaming_tool_adds_beta_and_strips_key():
|
||||
result = _invoke_request_with_tools(
|
||||
[
|
||||
_eager_invoke_tool("write_file", True),
|
||||
_eager_invoke_tool("read_file", False),
|
||||
{"name": "list_files", "input_schema": {"type": "object", "properties": {}}},
|
||||
]
|
||||
)
|
||||
|
||||
assert result["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA]
|
||||
assert [tool["name"] for tool in result["tools"]] == ["write_file", "read_file", "list_files"]
|
||||
assert all("eager_input_streaming" not in tool for tool in result["tools"])
|
||||
assert result["tools"][0]["description"] == "write_file tool"
|
||||
assert result["tools"][0]["input_schema"] == {"type": "object", "properties": {"path": {"type": "string"}}}
|
||||
|
||||
|
||||
def test_bedrock_invoke_eager_input_streaming_false_strips_key_without_beta():
|
||||
result = _invoke_request_with_tools([_eager_invoke_tool("write_file", False)])
|
||||
|
||||
assert "anthropic_beta" not in result
|
||||
assert result["tools"] == [
|
||||
{
|
||||
"name": "write_file",
|
||||
"description": "write_file tool",
|
||||
"input_schema": {"type": "object", "properties": {"path": {"type": "string"}}},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_bedrock_invoke_eager_input_streaming_beta_not_duplicated_with_client_header():
|
||||
result = _invoke_request_with_tools(
|
||||
[_eager_invoke_tool("write_file", True)],
|
||||
headers={"anthropic-beta": FINE_GRAINED_TOOL_STREAMING_BETA},
|
||||
)
|
||||
|
||||
assert result["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA]
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -25970,6 +25970,8 @@ export interface components {
|
|||
/** Allowed Callers */
|
||||
allowed_callers?: string[];
|
||||
cache_control?: components["schemas"]["ChatCompletionCachedContent"];
|
||||
/** Eager Input Streaming */
|
||||
eager_input_streaming?: boolean;
|
||||
function: components["schemas"]["ChatCompletionToolParamFunctionChunk"];
|
||||
/** Type */
|
||||
type: "function" | string;
|
||||
|
|
@ -25978,6 +25980,8 @@ export interface components {
|
|||
ChatCompletionToolParamFunctionChunk: {
|
||||
/** Description */
|
||||
description?: string;
|
||||
/** Eager Input Streaming */
|
||||
eager_input_streaming?: boolean;
|
||||
/** Name */
|
||||
name: string;
|
||||
/** Parameters */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue