mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
Merge pull request #41871 from BerriAI/litellm_bedrock_eager_input_streaming
feat: honor eager_input_streaming on Bedrock and Anthropic Claude tools
This commit is contained in:
Mateo Wang
GitHub
commit
3490754e65
20 changed files with 570 additions and 55 deletions
|
|
@ -94,6 +94,7 @@ from litellm.utils import (
|
|||
from ..common_utils import (
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
eager_input_streaming_flag,
|
||||
process_anthropic_headers,
|
||||
strip_advisor_blocks_from_messages,
|
||||
)
|
||||
|
|
@ -732,10 +733,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
|
||||
input_anthropic_schema: Final = sanitize_input_schema_for_anthropic(_input_schema)
|
||||
|
||||
_tool: Final = AnthropicMessagesTool(
|
||||
name=tool["function"]["name"],
|
||||
input_schema=input_anthropic_schema,
|
||||
type="custom",
|
||||
_eager_input_streaming: Final = eager_input_streaming_flag(tool)
|
||||
_tool: Final = (
|
||||
AnthropicMessagesTool(
|
||||
name=tool["function"]["name"],
|
||||
input_schema=input_anthropic_schema,
|
||||
type="custom",
|
||||
)
|
||||
if _eager_input_streaming is None
|
||||
else AnthropicMessagesTool(
|
||||
name=tool["function"]["name"],
|
||||
input_schema=input_anthropic_schema,
|
||||
type="custom",
|
||||
eager_input_streaming=_eager_input_streaming,
|
||||
)
|
||||
)
|
||||
|
||||
_description: Final = tool["function"].get("description")
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from types import MappingProxyType
|
|||
from typing import Any, Final, Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
|
|
@ -19,6 +19,7 @@ from litellm.constants import (
|
|||
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
|
||||
)
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_file_ids_from_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
|
|
@ -231,6 +232,27 @@ def optionally_handle_anthropic_oauth(headers: dict, api_key: str | None) -> tup
|
|||
return headers, api_key
|
||||
|
||||
|
||||
class _EagerInputStreamingFunction(BaseModel):
|
||||
eager_input_streaming: StrictBool | None = None
|
||||
|
||||
|
||||
class _EagerInputStreamingTool(BaseModel):
|
||||
eager_input_streaming: StrictBool | None = None
|
||||
function: _EagerInputStreamingFunction | None = None
|
||||
|
||||
|
||||
def eager_input_streaming_flag(tool: object) -> bool | None:
|
||||
try:
|
||||
parsed: Final = _EagerInputStreamingTool.model_validate(tool)
|
||||
except ValidationError as error:
|
||||
if isinstance(tool, Mapping):
|
||||
raise UnsupportedParamsError(message="eager_input_streaming must be a boolean") from error
|
||||
return None
|
||||
if parsed.eager_input_streaming is not None:
|
||||
return parsed.eager_input_streaming
|
||||
return parsed.function.eager_input_streaming if parsed.function is not None else None
|
||||
|
||||
|
||||
class AnthropicError(BaseLLMException):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -373,6 +395,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
|
||||
return False
|
||||
|
||||
def is_eager_input_streaming_used(self, tools: Sequence[object] | None) -> bool:
|
||||
return any(eager_input_streaming_flag(tool) is True for tool in tools or ())
|
||||
|
||||
@staticmethod
|
||||
def _supports_sampling_params(model: str) -> bool:
|
||||
"""Claude 4.7+ (Opus 4.7/4.8, Fable 5) removed sampling params: the API
|
||||
|
|
|
|||
|
|
@ -111,6 +111,7 @@ from litellm.litellm_core_utils.reasoning_effort_utils import (
|
|||
reasoning_effort_from_thinking_budget,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
eager_input_streaming_flag,
|
||||
is_empty_unsigned_thinking_block,
|
||||
normalize_anthropic_tool_use_id,
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
|
|
@ -197,6 +198,15 @@ def target_supports_mid_conversation_system(model: str | None, custom_llm_provid
|
|||
return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
|
||||
def _chat_tool_param(function_chunk: ChatCompletionToolParamFunctionChunk, tool: object) -> ChatCompletionToolParam:
|
||||
eager_input_streaming: Final = eager_input_streaming_flag(tool)
|
||||
if eager_input_streaming is None:
|
||||
return ChatCompletionToolParam(type="function", function=function_chunk)
|
||||
return ChatCompletionToolParam(
|
||||
type="function", function=function_chunk, eager_input_streaming=eager_input_streaming
|
||||
)
|
||||
|
||||
|
||||
class AnthropicAdapter:
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
|
@ -770,6 +780,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
"cache_control",
|
||||
"strict",
|
||||
"type",
|
||||
"eager_input_streaming",
|
||||
]
|
||||
|
||||
for idx, tool in enumerate(tools):
|
||||
|
|
@ -808,7 +819,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
for k, v in tool.items():
|
||||
if k not in mapped_tool_params: # pass additional computer kwargs
|
||||
function_chunk.setdefault("parameters", {}).update({k: v})
|
||||
tool_param = ChatCompletionToolParam(type="function", function=function_chunk)
|
||||
tool_param = _chat_tool_param(function_chunk, tool)
|
||||
self._add_cache_control_if_applicable(tool, tool_param, model)
|
||||
new_tools.append(tool_param)
|
||||
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ from litellm.llms.bedrock.request_metadata import (
|
|||
merge_bedrock_invoke_headers,
|
||||
resolve_bedrock_request_metadata,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER
|
||||
from litellm.types.llms.bedrock import *
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -1517,12 +1518,6 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""Process tools and collect anthropic_beta values."""
|
||||
bedrock_tools: list[ToolBlock] = []
|
||||
|
||||
# Collect anthropic_beta values from user headers
|
||||
anthropic_beta_list: Final = []
|
||||
if headers:
|
||||
user_betas: Final = get_anthropic_beta_from_headers(headers)
|
||||
anthropic_beta_list.extend(user_betas)
|
||||
|
||||
# Separate pre-formatted Bedrock tools (e.g. systemTool from web_search_options)
|
||||
# from OpenAI-format tools that need transformation via _bedrock_tools_pt
|
||||
filtered_tools: Final = []
|
||||
|
|
@ -1542,6 +1537,17 @@ class AmazonConverseConfig(BaseConfig):
|
|||
continue
|
||||
filtered_tools.append(tool)
|
||||
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
client_beta_list: Final = get_anthropic_beta_from_headers(headers or {})
|
||||
eager_beta: Final = (
|
||||
(ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,)
|
||||
if base_model.startswith("anthropic")
|
||||
and AnthropicModelInfo().is_eager_input_streaming_used(filtered_tools)
|
||||
and ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER not in client_beta_list
|
||||
else ()
|
||||
)
|
||||
anthropic_beta_list: Final = [*client_beta_list, *eager_beta]
|
||||
|
||||
# Only separate tools if computer use tools are actually present
|
||||
if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model):
|
||||
# Separate computer use tools from regular function tools
|
||||
|
|
@ -1619,7 +1625,6 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
# Opus 4.5 gates ``output_config.effort`` behind a beta header;
|
||||
# Claude 4.6/4.7 accept it without one.
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
if base_model.startswith("anthropic"):
|
||||
output_config: Final = additional_request_params.get("output_config")
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -24,8 +24,12 @@ from litellm.llms.bedrock.common_utils import (
|
|||
normalize_custom_field_on_tools,
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke,
|
||||
strip_unsupported_bedrock_invoke_output_config_keys,
|
||||
tools_without_eager_input_streaming,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,
|
||||
ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
|
@ -237,6 +241,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
# Hoist `custom.defer_loading` then drop `custom` (Bedrock doesn't support it)
|
||||
normalize_custom_field_on_tools(anthropic_request)
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke(anthropic_request)
|
||||
outbound_tools: Final = tools_without_eager_input_streaming(anthropic_request)
|
||||
if outbound_tools is not None:
|
||||
anthropic_request["tools"] = outbound_tools
|
||||
return anthropic_request
|
||||
|
||||
def _compute_bedrock_invoke_beta_headers(
|
||||
|
|
@ -269,6 +276,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
if bedrock_supports_tool_search(model):
|
||||
beta_set.add("tool-search-tool-2025-10-19")
|
||||
|
||||
if self.is_eager_input_streaming_used(tools):
|
||||
beta_set.add(ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER)
|
||||
|
||||
auto_beta_list: Final = filter_and_transform_beta_headers(
|
||||
beta_headers=list(beta_set - user_beta_set),
|
||||
provider="bedrock",
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import functools
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -18,6 +18,7 @@ if TYPE_CHECKING:
|
|||
from litellm.types.llms.bedrock import BedrockCreateBatchRequest
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -330,6 +331,17 @@ def normalize_custom_field_on_tools(request_body: dict) -> None:
|
|||
tool["defer_loading"] = deferred
|
||||
|
||||
|
||||
_TOOL_DICTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
def tools_without_eager_input_streaming(request_body: Mapping[str, object]) -> Sequence[object] | None:
|
||||
try:
|
||||
tools: Final = _TOOL_DICTS_ADAPTER.validate_python(request_body.get("tools"))
|
||||
except ValidationError:
|
||||
return None
|
||||
return [{key: value for key, value in tool.items() if key != "eager_input_streaming"} for tool in tools]
|
||||
|
||||
|
||||
def normalize_json_schema_custom_types_to_object(schema: dict) -> None:
|
||||
"""
|
||||
In-place: replace JSON Schema ``type: \"custom\"`` with ``\"object\"`` (iterative walk).
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
normalize_custom_field_on_tools,
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke,
|
||||
strip_unsupported_bedrock_invoke_output_config_keys,
|
||||
tools_without_eager_input_streaming,
|
||||
)
|
||||
from litellm.llms.bedrock.request_metadata import (
|
||||
bedrock_request_metadata_headers,
|
||||
|
|
@ -46,6 +47,7 @@ from litellm.llms.bedrock.request_metadata import (
|
|||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,
|
||||
ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
|
||||
)
|
||||
from litellm.types.llms.bedrock import BedrockInvokeAnthropicMessagesRequest
|
||||
|
|
@ -525,6 +527,9 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
if injected_thinking_for_clear_thinking:
|
||||
beta_set.add("interleaved-thinking-2025-05-14")
|
||||
|
||||
if anthropic_model_info.is_eager_input_streaming_used(tools):
|
||||
beta_set.add(ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER)
|
||||
|
||||
self._filter_context_management_for_bedrock_invoke(
|
||||
anthropic_messages_request=anthropic_messages_request,
|
||||
beta_set=beta_set,
|
||||
|
|
@ -719,6 +724,10 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
if filtered_betas:
|
||||
anthropic_messages_request["anthropic_beta"] = filtered_betas
|
||||
|
||||
outbound_tools: Final = tools_without_eager_input_streaming(anthropic_messages_request)
|
||||
if outbound_tools is not None:
|
||||
anthropic_messages_request["tools"] = outbound_tools
|
||||
|
||||
remaining_output_config: Final = anthropic_messages_request.get("output_config")
|
||||
if (
|
||||
litellm.drop_params is True
|
||||
|
|
|
|||
|
|
@ -1146,37 +1146,35 @@ def responses_api_bridge_check(
|
|||
return model_info, model
|
||||
|
||||
|
||||
def _should_allow_input_examples(custom_llm_provider: str | None, model: str) -> bool:
|
||||
_ANTHROPIC_ONLY_TOOL_KEYS: Final = frozenset({"input_examples", "eager_input_streaming"})
|
||||
|
||||
|
||||
def _is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool:
|
||||
if custom_llm_provider == "anthropic":
|
||||
return True
|
||||
if custom_llm_provider == "azure_ai" or custom_llm_provider == "bedrock" or custom_llm_provider == "vertex_ai":
|
||||
return "claude" in model.lower()
|
||||
model_lower: Final = model.lower()
|
||||
if custom_llm_provider == "bedrock":
|
||||
return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower)
|
||||
if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai":
|
||||
return "claude" in model_lower
|
||||
return False
|
||||
|
||||
|
||||
def _drop_input_examples_from_tool(tool: dict) -> dict:
|
||||
tool_copy: Final = tool.copy()
|
||||
tool_copy.pop("input_examples", None)
|
||||
function = tool_copy.get("function")
|
||||
if isinstance(function, dict):
|
||||
function = function.copy()
|
||||
function.pop("input_examples", None)
|
||||
tool_copy["function"] = function
|
||||
return tool_copy
|
||||
def _without_anthropic_only_tool_keys(tool: dict) -> dict:
|
||||
kept: Final = {key: value for key, value in tool.items() if key not in _ANTHROPIC_ONLY_TOOL_KEYS}
|
||||
function: Final = tool.get("function")
|
||||
if not isinstance(function, dict):
|
||||
return kept
|
||||
return {
|
||||
**kept,
|
||||
"function": {key: value for key, value in function.items() if key not in _ANTHROPIC_ONLY_TOOL_KEYS},
|
||||
}
|
||||
|
||||
|
||||
def _drop_input_examples_from_tools(
|
||||
tools: list[dict] | None,
|
||||
) -> list[dict] | None:
|
||||
def _drop_anthropic_only_tool_keys(tools: list[dict] | None) -> list[dict] | None:
|
||||
if tools is None:
|
||||
return None
|
||||
cleaned_tools: Final[list[dict]] = []
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict):
|
||||
cleaned_tools.append(_drop_input_examples_from_tool(tool))
|
||||
else:
|
||||
cleaned_tools.append(tool)
|
||||
return cleaned_tools
|
||||
return [_without_anthropic_only_tool_keys(tool) if isinstance(tool, dict) else tool for tool in tools]
|
||||
|
||||
|
||||
class _ProxyAuthHeadersProvider(Protocol):
|
||||
|
|
@ -5360,8 +5358,8 @@ def completion(
|
|||
api_base=api_base,
|
||||
)
|
||||
|
||||
if not _should_allow_input_examples(custom_llm_provider=custom_llm_provider, model=model):
|
||||
tools = _drop_input_examples_from_tools(tools=tools)
|
||||
if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model):
|
||||
tools = _drop_anthropic_only_tool_keys(tools=tools)
|
||||
|
||||
if provider_specific_header is not None:
|
||||
headers.update(
|
||||
|
|
|
|||
|
|
@ -33378,6 +33378,10 @@
|
|||
"cache_control": {
|
||||
"$ref": "#/components/schemas/ChatCompletionCachedContent"
|
||||
},
|
||||
"eager_input_streaming": {
|
||||
"title": "Eager Input Streaming",
|
||||
"type": "boolean"
|
||||
},
|
||||
"function": {
|
||||
"$ref": "#/components/schemas/ChatCompletionToolParamFunctionChunk"
|
||||
},
|
||||
|
|
@ -33407,6 +33411,10 @@
|
|||
"title": "Description",
|
||||
"type": "string"
|
||||
},
|
||||
"eager_input_streaming": {
|
||||
"title": "Eager Input Streaming",
|
||||
"type": "boolean"
|
||||
},
|
||||
"name": {
|
||||
"title": "Name",
|
||||
"type": "string"
|
||||
|
|
|
|||
|
|
@ -2020,6 +2020,8 @@ class LiteLLMCompletionResponsesConfig:
|
|||
chat_completion_tool["allowed_callers"] = tool.get("allowed_callers")
|
||||
if tool.get("input_examples"):
|
||||
chat_completion_tool["input_examples"] = tool.get("input_examples")
|
||||
if tool.get("eager_input_streaming") is not None:
|
||||
chat_completion_tool["eager_input_streaming"] = tool.get("eager_input_streaming")
|
||||
return ResponsesToolChatForm(
|
||||
chat_tools=(cast(ChatCompletionToolParam, chat_completion_tool),), web_search_options=None
|
||||
)
|
||||
|
|
@ -2096,6 +2098,8 @@ class LiteLLMCompletionResponsesConfig:
|
|||
responses_tool["allowed_callers"] = tool.get("allowed_callers")
|
||||
if tool.get("input_examples") is not None:
|
||||
responses_tool["input_examples"] = tool.get("input_examples")
|
||||
if tool.get("eager_input_streaming") is not None:
|
||||
responses_tool["eager_input_streaming"] = tool.get("eager_input_streaming")
|
||||
result.append(responses_tool)
|
||||
else:
|
||||
# mcp or other: pass through unchanged
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ class AnthropicMessagesTool(TypedDict, total=False):
|
|||
defer_loading: bool
|
||||
allowed_callers: list[str] | None
|
||||
input_examples: list[dict[str, Any]] | None
|
||||
eager_input_streaming: ReadOnly[bool]
|
||||
|
||||
|
||||
class AnthropicComputerTool(TypedDict, total=False):
|
||||
|
|
@ -755,6 +756,8 @@ ANTHROPIC_TOOL_SEARCH_BETA_HEADER: Final = "advanced-tool-use-2025-11-20"
|
|||
# Effort beta header constant
|
||||
ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24"
|
||||
|
||||
ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER: Final = "fine-grained-tool-streaming-2025-05-14"
|
||||
|
||||
# OAuth constants
|
||||
ANTHROPIC_OAUTH_TOKEN_PREFIX: Final = "sk-ant-oat"
|
||||
ANTHROPIC_OAUTH_BETA_HEADER: Final = "oauth-2025-04-20"
|
||||
|
|
|
|||
|
|
@ -992,6 +992,7 @@ class ChatCompletionToolParamFunctionChunk(TypedDict, total=False):
|
|||
description: str
|
||||
parameters: dict
|
||||
strict: bool
|
||||
eager_input_streaming: ReadOnly[bool]
|
||||
|
||||
|
||||
class OpenAIChatCompletionToolParam(TypedDict):
|
||||
|
|
@ -1002,6 +1003,7 @@ class OpenAIChatCompletionToolParam(TypedDict):
|
|||
class ChatCompletionToolParam(OpenAIChatCompletionToolParam, total=False):
|
||||
cache_control: ChatCompletionCachedContent
|
||||
allowed_callers: list[str]
|
||||
eager_input_streaming: ReadOnly[bool]
|
||||
|
||||
|
||||
class Function(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -6370,3 +6370,74 @@ def test_response_format_tool_path_skips_forced_tool_choice_when_unsupported(loc
|
|||
|
||||
assert "tools" in result
|
||||
assert "tool_choice" not in result
|
||||
|
||||
|
||||
def _eager_chat_function(**extra: object) -> dict[str, object]:
|
||||
return {
|
||||
"name": "write_file",
|
||||
"description": "Write a file",
|
||||
"parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]},
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def _eager_chat_tool(**extra: object) -> dict[str, object]:
|
||||
return {"type": "function", "function": _eager_chat_function(), **extra}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", [True, False])
|
||||
def test_eager_input_streaming_passed_through_from_tool_top_level(flag):
|
||||
mapped_tool, _ = AnthropicConfig()._map_tool_helper(_eager_chat_tool(eager_input_streaming=flag))
|
||||
|
||||
assert mapped_tool == {
|
||||
"name": "write_file",
|
||||
"description": "Write a file",
|
||||
"input_schema": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]},
|
||||
"type": "custom",
|
||||
"eager_input_streaming": flag,
|
||||
}
|
||||
|
||||
|
||||
def test_eager_input_streaming_passed_through_from_function():
|
||||
mapped_tool, _ = AnthropicConfig()._map_tool_helper(
|
||||
{"type": "function", "function": _eager_chat_function(eager_input_streaming=True)}
|
||||
)
|
||||
|
||||
assert mapped_tool["eager_input_streaming"] is True
|
||||
assert "eager_input_streaming" not in mapped_tool["input_schema"]
|
||||
|
||||
|
||||
def test_eager_input_streaming_absent_stays_absent():
|
||||
mapped_tool, _ = AnthropicConfig()._map_tool_helper(_eager_chat_tool())
|
||||
|
||||
assert "eager_input_streaming" not in mapped_tool
|
||||
|
||||
|
||||
def test_eager_input_streaming_rejects_non_boolean():
|
||||
with pytest.raises(litellm.BadRequestError, match="eager_input_streaming must be a boolean"):
|
||||
AnthropicConfig()._map_tool_helper(_eager_chat_tool(eager_input_streaming="true"))
|
||||
|
||||
|
||||
def test_eager_input_streaming_not_set_on_computer_use_tool():
|
||||
computer_tool = {
|
||||
"type": "computer_20250124",
|
||||
"function": {"name": "computer", "parameters": {"display_width_px": 1024, "display_height_px": 768}},
|
||||
"eager_input_streaming": True,
|
||||
}
|
||||
|
||||
mapped_tool, _ = AnthropicConfig()._map_tool_helper(computer_tool)
|
||||
|
||||
assert mapped_tool["type"] == "computer_20250124"
|
||||
assert "eager_input_streaming" not in mapped_tool
|
||||
|
||||
|
||||
def test_eager_input_streaming_reaches_anthropic_request_tools():
|
||||
result = AnthropicConfig().map_openai_params(
|
||||
non_default_params={"tools": [_eager_chat_tool(eager_input_streaming=True)], "stream": True},
|
||||
optional_params={},
|
||||
model="claude-sonnet-5",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["tools"][0]["eager_input_streaming"] is True
|
||||
assert result["tools"][0]["name"] == "write_file"
|
||||
|
|
|
|||
|
|
@ -5017,3 +5017,45 @@ def test_redacted_thinking_blocks_never_carry_cache_control():
|
|||
replayed: Final = outbound["messages"][1]["content"][0]
|
||||
assert replayed["type"] == "redacted_thinking"
|
||||
assert "cache_control" not in replayed
|
||||
|
||||
|
||||
EAGER_INPUT_SCHEMA: Final = {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", [True, False])
|
||||
def test_translate_anthropic_tools_to_openai_carries_eager_input_streaming_onto_tool(flag):
|
||||
"""The per-tool flag lands on the OpenAI tool object, never inside the JSON schema Bedrock sends as inputSchema."""
|
||||
tools: Final = [{"name": "write_file", "input_schema": EAGER_INPUT_SCHEMA, "eager_input_streaming": flag}]
|
||||
|
||||
new_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(tools=tools)
|
||||
|
||||
assert new_tools[0]["eager_input_streaming"] is flag
|
||||
assert new_tools[0]["function"]["parameters"] == EAGER_INPUT_SCHEMA
|
||||
assert "eager_input_streaming" not in new_tools[0]["function"]
|
||||
|
||||
|
||||
def test_translate_anthropic_tools_to_openai_omits_unset_eager_input_streaming():
|
||||
tools: Final = [{"name": "write_file", "input_schema": EAGER_INPUT_SCHEMA}]
|
||||
|
||||
new_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(tools=tools)
|
||||
|
||||
assert "eager_input_streaming" not in new_tools[0]
|
||||
assert "eager_input_streaming" not in new_tools[0]["function"]["parameters"]
|
||||
|
||||
|
||||
def test_eager_input_streaming_tool_reaches_bedrock_converse_as_beta():
|
||||
"""An Anthropic Messages request routed to bedrock/converse/ turns the flag into the fine-grained streaming beta."""
|
||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||
|
||||
tools: Final = [{"name": "write_file", "input_schema": EAGER_INPUT_SCHEMA, "eager_input_streaming": True}]
|
||||
new_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(tools=tools)
|
||||
|
||||
data: Final = AmazonConverseConfig()._transform_request_helper(
|
||||
model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
system_content_blocks=[],
|
||||
optional_params={"tools": new_tools},
|
||||
messages=[{"role": "user", "content": "write a big file"}],
|
||||
)
|
||||
|
||||
assert data["additionalModelRequestFields"]["anthropic_beta"] == ["fine-grained-tool-streaming-2025-05-14"]
|
||||
assert data["toolConfig"]["tools"][0]["toolSpec"]["inputSchema"]["json"] == EAGER_INPUT_SCHEMA
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -859,3 +860,58 @@ def test_bedrock_chat_invoke_tool_search_beta_follows_model_map(
|
|||
)
|
||||
|
||||
assert result.get("anthropic_beta") == expected_betas
|
||||
|
||||
|
||||
FINE_GRAINED_TOOL_STREAMING_BETA: Final = "fine-grained-tool-streaming-2025-05-14"
|
||||
EAGER_TOOL_SCHEMA: Final = {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}
|
||||
|
||||
|
||||
def _chat_invoke_request_with_tools(
|
||||
tools: list[dict[str, object]], headers: dict[str, str] | None = None
|
||||
) -> dict[str, object]:
|
||||
config: Final = AmazonAnthropicClaudeConfig()
|
||||
model: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
optional_params: Final = config.map_openai_params(
|
||||
non_default_params={"max_tokens": 64, "stream": True, "tools": tools},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
return config.transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "write a big file"}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers=headers or {},
|
||||
)
|
||||
|
||||
|
||||
def _eager_openai_tool(name: str, **extra: object) -> dict[str, object]:
|
||||
return {"type": "function", "function": {"name": name, "parameters": EAGER_TOOL_SCHEMA}, **extra}
|
||||
|
||||
|
||||
def test_bedrock_chat_invoke_eager_input_streaming_tool_adds_beta_and_strips_key():
|
||||
result = _chat_invoke_request_with_tools(
|
||||
[_eager_openai_tool("write_file", eager_input_streaming=True), _eager_openai_tool("read_file")]
|
||||
)
|
||||
|
||||
assert result["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA]
|
||||
assert [tool["name"] for tool in result["tools"]] == ["write_file", "read_file"]
|
||||
assert all("eager_input_streaming" not in tool for tool in result["tools"])
|
||||
assert result["tools"][0]["input_schema"] == EAGER_TOOL_SCHEMA
|
||||
|
||||
|
||||
def test_bedrock_chat_invoke_eager_input_streaming_false_strips_key_without_beta():
|
||||
result = _chat_invoke_request_with_tools([_eager_openai_tool("write_file", eager_input_streaming=False)])
|
||||
|
||||
assert "anthropic_beta" not in result
|
||||
assert "eager_input_streaming" not in result["tools"][0]
|
||||
|
||||
|
||||
def test_bedrock_chat_invoke_eager_input_streaming_beta_not_duplicated_with_client_header():
|
||||
result = _chat_invoke_request_with_tools(
|
||||
[_eager_openai_tool("write_file", eager_input_streaming=True)],
|
||||
headers={"anthropic-beta": FINE_GRAINED_TOOL_STREAMING_BETA},
|
||||
)
|
||||
|
||||
assert result["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA]
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import os
|
|||
import httpx
|
||||
import pytest
|
||||
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import litellm
|
||||
|
|
@ -7400,3 +7401,111 @@ def test_transform_response_honors_json_mode_kwarg_when_optional_params_lack_it(
|
|||
)
|
||||
assert result.choices[0].message.tool_calls is None
|
||||
assert json.loads(result.choices[0].message.content) == {"city": "Paris", "population": 2100000}
|
||||
|
||||
|
||||
FINE_GRAINED_TOOL_STREAMING_BETA: Final = "fine-grained-tool-streaming-2025-05-14"
|
||||
EAGER_TOOL_SCHEMA: Final = {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}
|
||||
|
||||
|
||||
def _eager_openai_tool(**extra: object) -> dict[str, object]:
|
||||
return {"type": "function", "function": {"name": "write_file", "parameters": EAGER_TOOL_SCHEMA}, **extra}
|
||||
|
||||
|
||||
def _eager_openai_function_tool(**extra: object) -> dict[str, object]:
|
||||
return {"type": "function", "function": {"name": "write_file", "parameters": EAGER_TOOL_SCHEMA, **extra}}
|
||||
|
||||
|
||||
def _eager_anthropic_tool(**extra: object) -> dict[str, object]:
|
||||
return {"name": "write_file", "input_schema": EAGER_TOOL_SCHEMA, **extra}
|
||||
|
||||
|
||||
def _converse_request(
|
||||
model: str, tools: list[dict[str, object]], headers: dict[str, object] | None = None
|
||||
) -> dict[str, object]:
|
||||
return AmazonConverseConfig()._transform_request_helper(
|
||||
model=model,
|
||||
system_content_blocks=[],
|
||||
optional_params={"tools": tools},
|
||||
messages=[{"role": "user", "content": "write a big file"}],
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
[
|
||||
_eager_openai_tool(eager_input_streaming=True),
|
||||
_eager_openai_function_tool(eager_input_streaming=True),
|
||||
_eager_anthropic_tool(eager_input_streaming=True),
|
||||
],
|
||||
ids=["openai_top_level", "openai_under_function", "anthropic_shape"],
|
||||
)
|
||||
def test_eager_input_streaming_tool_adds_fine_grained_tool_streaming_beta(tool):
|
||||
data = _converse_request("us.anthropic.claude-sonnet-4-5-20250929-v1:0", [tool])
|
||||
|
||||
assert data["additionalModelRequestFields"]["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA]
|
||||
tool_spec = data["toolConfig"]["tools"][0]["toolSpec"]
|
||||
assert tool_spec["name"] == "write_file"
|
||||
assert "eager_input_streaming" not in tool_spec
|
||||
assert "eager_input_streaming" not in tool_spec["inputSchema"]["json"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
[
|
||||
_eager_openai_tool(eager_input_streaming=False),
|
||||
_eager_openai_function_tool(eager_input_streaming=False),
|
||||
_eager_anthropic_tool(eager_input_streaming=False),
|
||||
_eager_openai_tool(),
|
||||
],
|
||||
ids=["openai_false", "function_false", "anthropic_false", "absent"],
|
||||
)
|
||||
def test_eager_input_streaming_false_or_absent_adds_no_beta(tool):
|
||||
data = _converse_request("us.anthropic.claude-sonnet-4-5-20250929-v1:0", [tool])
|
||||
|
||||
assert "anthropic_beta" not in data.get("additionalModelRequestFields", {})
|
||||
assert "eager_input_streaming" not in data["toolConfig"]["tools"][0]["toolSpec"]
|
||||
|
||||
|
||||
def test_eager_input_streaming_beta_only_on_anthropic_models():
|
||||
data = _converse_request("amazon.nova-pro-v1:0", [_eager_openai_tool(eager_input_streaming=True)])
|
||||
|
||||
assert "anthropic_beta" not in data.get("additionalModelRequestFields", {})
|
||||
assert data["toolConfig"]["tools"][0]["toolSpec"]["name"] == "write_file"
|
||||
|
||||
|
||||
def test_eager_input_streaming_beta_not_duplicated_with_client_header():
|
||||
data = _converse_request(
|
||||
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
[_eager_openai_tool(eager_input_streaming=True)],
|
||||
headers={"anthropic-beta": f"{FINE_GRAINED_TOOL_STREAMING_BETA},interleaved-thinking-2025-05-14"},
|
||||
)
|
||||
|
||||
assert data["additionalModelRequestFields"]["anthropic_beta"] == [
|
||||
FINE_GRAINED_TOOL_STREAMING_BETA,
|
||||
"interleaved-thinking-2025-05-14",
|
||||
]
|
||||
|
||||
|
||||
def test_eager_input_streaming_beta_never_written_back_into_client_header_list():
|
||||
headers = {"anthropic-beta": ["interleaved-thinking-2025-05-14"]}
|
||||
|
||||
data = _converse_request(
|
||||
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
[_eager_openai_tool(eager_input_streaming=True)],
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
assert data["additionalModelRequestFields"]["anthropic_beta"] == [
|
||||
"interleaved-thinking-2025-05-14",
|
||||
FINE_GRAINED_TOOL_STREAMING_BETA,
|
||||
]
|
||||
assert headers == {"anthropic-beta": ["interleaved-thinking-2025-05-14"]}
|
||||
|
||||
|
||||
def test_eager_input_streaming_non_boolean_is_a_bad_request():
|
||||
with pytest.raises(litellm.BadRequestError, match="eager_input_streaming must be a boolean"):
|
||||
_converse_request(
|
||||
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
[_eager_openai_tool(eager_input_streaming="true")],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import json
|
|||
import os
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -3244,3 +3245,67 @@ def test_bedrock_messages_strips_effort_but_keeps_format_for_sonnet_4_5(local_mo
|
|||
)
|
||||
|
||||
assert result.get("output_config") == {"format": schema_format}
|
||||
|
||||
|
||||
FINE_GRAINED_TOOL_STREAMING_BETA: Final = "fine-grained-tool-streaming-2025-05-14"
|
||||
|
||||
|
||||
def _invoke_request_with_tools(
|
||||
tools: list[dict[str, object]], headers: dict[str, str] | None = None
|
||||
) -> dict[str, object]:
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
return AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
|
||||
model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
messages=[{"role": "user", "content": "write a big file"}],
|
||||
anthropic_messages_optional_request_params={"max_tokens": 4096, "tools": copy.deepcopy(tools), "stream": True},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers=headers or {},
|
||||
)
|
||||
|
||||
|
||||
def _eager_invoke_tool(name: str, eager_input_streaming: bool) -> dict[str, object]:
|
||||
return {
|
||||
"name": name,
|
||||
"description": f"{name} tool",
|
||||
"input_schema": {"type": "object", "properties": {"path": {"type": "string"}}},
|
||||
"eager_input_streaming": eager_input_streaming,
|
||||
}
|
||||
|
||||
|
||||
def test_bedrock_invoke_eager_input_streaming_tool_adds_beta_and_strips_key():
|
||||
result = _invoke_request_with_tools(
|
||||
[
|
||||
_eager_invoke_tool("write_file", True),
|
||||
_eager_invoke_tool("read_file", False),
|
||||
{"name": "list_files", "input_schema": {"type": "object", "properties": {}}},
|
||||
]
|
||||
)
|
||||
|
||||
assert result["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA]
|
||||
assert [tool["name"] for tool in result["tools"]] == ["write_file", "read_file", "list_files"]
|
||||
assert all("eager_input_streaming" not in tool for tool in result["tools"])
|
||||
assert result["tools"][0]["description"] == "write_file tool"
|
||||
assert result["tools"][0]["input_schema"] == {"type": "object", "properties": {"path": {"type": "string"}}}
|
||||
|
||||
|
||||
def test_bedrock_invoke_eager_input_streaming_false_strips_key_without_beta():
|
||||
result = _invoke_request_with_tools([_eager_invoke_tool("write_file", False)])
|
||||
|
||||
assert "anthropic_beta" not in result
|
||||
assert result["tools"] == [
|
||||
{
|
||||
"name": "write_file",
|
||||
"description": "write_file tool",
|
||||
"input_schema": {"type": "object", "properties": {"path": {"type": "string"}}},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_bedrock_invoke_eager_input_streaming_beta_not_duplicated_with_client_header():
|
||||
result = _invoke_request_with_tools(
|
||||
[_eager_invoke_tool("write_file", True)],
|
||||
headers={"anthropic-beta": FINE_GRAINED_TOOL_STREAMING_BETA},
|
||||
)
|
||||
|
||||
assert result["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA]
|
||||
|
|
|
|||
|
|
@ -1900,6 +1900,38 @@ class TestToolTransformation:
|
|||
assert "defer_loading" not in result_tool
|
||||
assert "allowed_callers" not in result_tool
|
||||
assert "input_examples" not in result_tool
|
||||
assert "eager_input_streaming" not in result_tool
|
||||
|
||||
@pytest.mark.parametrize("eager_input_streaming", [True, False])
|
||||
def test_transform_function_tools_forwards_eager_input_streaming(self, eager_input_streaming: bool) -> None:
|
||||
function_tool: Final = {
|
||||
"type": "function",
|
||||
"name": "write_file",
|
||||
"parameters": {"type": "object", "properties": {"path": {"type": "string"}}},
|
||||
"eager_input_streaming": eager_input_streaming,
|
||||
}
|
||||
|
||||
result_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
|
||||
tools=[function_tool]
|
||||
)
|
||||
|
||||
assert result_tools[0]["eager_input_streaming"] is eager_input_streaming
|
||||
|
||||
@pytest.mark.parametrize("eager_input_streaming", [True, False])
|
||||
def test_chat_completion_tools_to_responses_tools_keeps_eager_input_streaming(
|
||||
self, eager_input_streaming: bool
|
||||
) -> None:
|
||||
chat_tool: Final = {
|
||||
"type": "function",
|
||||
"function": {"name": "write_file", "parameters": {"type": "object"}},
|
||||
"eager_input_streaming": eager_input_streaming,
|
||||
}
|
||||
|
||||
result_tools: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools(
|
||||
[chat_tool]
|
||||
)
|
||||
|
||||
assert result_tools[0]["eager_input_streaming"] is eager_input_streaming
|
||||
|
||||
def test_transform_code_execution_tools(self):
|
||||
"""Test that code_execution tools are passed through as-is"""
|
||||
|
|
|
|||
|
|
@ -349,28 +349,66 @@ def test_bedrock_latency_optimized_inference():
|
|||
assert json_data["performanceConfig"]["latency"] == "optimized"
|
||||
|
||||
|
||||
def test_strip_input_examples_for_non_anthropic_providers():
|
||||
@pytest.mark.parametrize(
|
||||
("custom_llm_provider", "model", "expected"),
|
||||
[
|
||||
("anthropic", "claude-sonnet-5", True),
|
||||
("bedrock", "us.anthropic.claude-sonnet-5-20260501-v1:0", True),
|
||||
("bedrock", "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", True),
|
||||
("bedrock", "us.amazon.nova-2-lite-v1:0", False),
|
||||
("vertex_ai", "claude-sonnet-5", True),
|
||||
("vertex_ai", "gemini-3.8-flash", False),
|
||||
("azure_ai", "claude-sonnet-4-6", True),
|
||||
("azure_ai", "gpt-5.6", False),
|
||||
("openai", "gpt-5.6", False),
|
||||
("gemini", "gemini-3.8-flash", False),
|
||||
],
|
||||
)
|
||||
def test_is_claude_tool_target(custom_llm_provider: str, model: str, expected: bool):
|
||||
assert litellm_main._is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", ["input_examples", "eager_input_streaming"])
|
||||
def test_drop_anthropic_only_tool_keys_strips_tool_and_function_levels(key: str):
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "example_tool",
|
||||
"input_examples": [{"foo": "bar"}],
|
||||
"function": {
|
||||
"name": "example_tool",
|
||||
"input_examples": [{"foo": "bar"}],
|
||||
},
|
||||
}
|
||||
{"type": "function", "name": "example_tool", key: True, "function": {"name": "example_tool", key: True}},
|
||||
"opaque_tool",
|
||||
]
|
||||
|
||||
assert not litellm_main._should_allow_input_examples(
|
||||
custom_llm_provider="openai", model="gpt-4o-mini"
|
||||
cleaned = litellm_main._drop_anthropic_only_tool_keys(tools=tools)
|
||||
|
||||
assert cleaned == [
|
||||
{"type": "function", "name": "example_tool", "function": {"name": "example_tool"}},
|
||||
"opaque_tool",
|
||||
]
|
||||
assert tools[0][key] is True
|
||||
assert tools[0]["function"][key] is True
|
||||
|
||||
|
||||
def test_completion_strips_eager_input_streaming_before_openai(respx_mock: respx.MockRouter, openai_api_response):
|
||||
api_base: Final = "http://localhost:12346/v1"
|
||||
mock_route: Final = respx_mock.post(url__regex=rf"{api_base}/chat/completions.*").mock(
|
||||
return_value=httpx.Response(status_code=200, json=openai_api_response)
|
||||
)
|
||||
|
||||
cleaned = litellm_main._drop_input_examples_from_tools(tools=tools)
|
||||
litellm.completion(
|
||||
model="openai/gpt-5.6",
|
||||
messages=[{"role": "user", "content": "Write the file"}],
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "write_file", "parameters": {"type": "object", "properties": {}}},
|
||||
"eager_input_streaming": True,
|
||||
}
|
||||
],
|
||||
api_base=api_base,
|
||||
api_key="fake_openai_api_key",
|
||||
)
|
||||
|
||||
assert isinstance(cleaned, list)
|
||||
assert "input_examples" not in cleaned[0]
|
||||
assert "input_examples" not in cleaned[0]["function"]
|
||||
assert mock_route.called
|
||||
sent_tool: Final = json.loads(respx_mock.calls[0].request.content)["tools"][0]
|
||||
assert "eager_input_streaming" not in sent_tool
|
||||
assert sent_tool["function"]["name"] == "write_file"
|
||||
|
||||
|
||||
def test_custom_provider_with_extra_headers():
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -26070,6 +26070,8 @@ export interface components {
|
|||
/** Allowed Callers */
|
||||
allowed_callers?: string[];
|
||||
cache_control?: components["schemas"]["ChatCompletionCachedContent"];
|
||||
/** Eager Input Streaming */
|
||||
eager_input_streaming?: boolean;
|
||||
function: components["schemas"]["ChatCompletionToolParamFunctionChunk"];
|
||||
/** Type */
|
||||
type: "function" | string;
|
||||
|
|
@ -26078,6 +26080,8 @@ export interface components {
|
|||
ChatCompletionToolParamFunctionChunk: {
|
||||
/** Description */
|
||||
description?: string;
|
||||
/** Eager Input Streaming */
|
||||
eager_input_streaming?: boolean;
|
||||
/** Name */
|
||||
name: string;
|
||||
/** Parameters */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue