mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
fix(anthropic): apply tool_config cache-control points on chat and /v1/messages (#47394)
* fix(anthropic): apply tool_config cache-control points on chat and /v1/messages
A cache_control_injection_points entry with location tool_config was spread into
the Anthropic request body on /v1/chat/completions, which Anthropic rejected with
400 Extra inputs are not permitted, and the native /v1/messages path dropped the
point silently. Both paths now put the point's control on the last tool Anthropic
caches and never forward the key.
Co-authored-by: Tan Nguyen <tan@pressingly.net>
* refactor(anthropic): validate tool_config inputs instead of casting them
Keeps the lint gates flat: the helper takes untyped tools and points and
narrows them with the existing object validators, the chat transform builds
the body with the marked tools instead of writing into optional_params, and
the handler reads a typed view of kwargs.
* fix(openai): keep tool_config cache points out of extra_body, add integration test
The cache hook hands a tool_config point on for the provider step, and for
providers that call through the OpenAI SDK every non-OpenAI param rides in
extra_body, so OpenAI answered 400 Unrecognized request argument supplied.
The new integration file drives a live proxy against scripted Anthropic,
Bedrock Converse and OpenAI upstreams for /v1/chat/completions and
/v1/messages.
* test(integration): cover tool_config cache points on bridged /v1/messages
Three new cases on /v1/messages for models without a native Anthropic
messages path: OpenAI through the Responses bridge and Gemini through the
chat bridge never see cache_control_injection_points in the body, and
Bedrock Nova through the chat bridge gets the cachePoint on toolConfig
---------
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Tan Nguyen <tan@pressingly.net>
(cherry picked from commit 93b79d2587)
This commit is contained in:
Kerry Lu
Yuneng Jiang
parent
c2788016b7
commit
6df738c397
9 changed files with 620 additions and 154 deletions
|
|
@ -113,6 +113,7 @@ from ..common_utils import (
|
|||
refusal_provider_specific_fields,
|
||||
requires_native_compaction_beta,
|
||||
strip_advisor_blocks_from_messages,
|
||||
tools_marked_for_a_tool_config_point,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -2357,6 +2358,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# Remove internal LiteLLM parameters that should not be sent to Anthropic API
|
||||
optional_params.pop("is_vertex_request", None)
|
||||
optional_params.pop("client_metadata", None)
|
||||
request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature
|
||||
dict[str, object], optional_params
|
||||
)
|
||||
marked_tools: Final = tools_marked_for_a_tool_config_point(
|
||||
request_params.get("tools"), request_params.pop("cache_control_injection_points", None)
|
||||
)
|
||||
if marked_tools is not None:
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
|
||||
request_litellm_params: Final = cast( # cast-ok: litellm_params is the untyped dict of the shared signature
|
||||
Mapping[str, object], litellm_params
|
||||
)
|
||||
AnthropicCacheControlHook.record_gateway_injection(request_litellm_params, 1)
|
||||
|
||||
# ``top_k`` is a provider-specific kwarg that bypasses
|
||||
# ``map_openai_params``; gate it here, the single boundary shared by
|
||||
|
|
@ -2376,6 +2390,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
"model": model,
|
||||
"messages": anthropic_messages,
|
||||
**optional_params,
|
||||
**({} if marked_tools is None else {"tools": list(marked_tools)}),
|
||||
}
|
||||
|
||||
self._apply_output_config(data=data, model=model, optional_params=optional_params)
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ from litellm.types.llms.anthropic import (
|
|||
ANTHROPIC_OAUTH_TOKEN_PREFIX,
|
||||
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
|
||||
AllAnthropicToolsValues,
|
||||
AnthropicMcpServerTool,
|
||||
AnthropicMessagesToolChoice,
|
||||
|
|
@ -111,7 +112,7 @@ _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
|
|||
_DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
|
||||
_CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
|
||||
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
|
||||
_CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
||||
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
|
|
@ -160,6 +161,40 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None
|
|||
)
|
||||
|
||||
|
||||
def _tool_config_point_control(points: object) -> Mapping[object, object] | None:
|
||||
point_fields: Final = tuple(validated_object_mapping(point) for point in validated_object_list(points) or ())
|
||||
tool_config_point: Final = next(
|
||||
(fields for fields in point_fields if fields is not None and fields.get("location") == "tool_config"), None
|
||||
)
|
||||
if tool_config_point is None:
|
||||
return None
|
||||
control: Final = tool_config_point.get("control")
|
||||
return {"type": "ephemeral"} if control is None else validated_object_mapping(control)
|
||||
|
||||
|
||||
def tools_marked_for_a_tool_config_point(tools: object, points: object) -> tuple[object, ...] | None:
|
||||
"""Put the ``tool_config`` point's breakpoint on the last tool Anthropic caches.
|
||||
|
||||
Anthropic caches tools as one prefix ending at the marked tool, so the last one carries
|
||||
the mark, skipping tool-search tools whose transform drops cache markers. A client that
|
||||
already marked a tool keeps its own breakpoint. Returns ``None`` when nothing is marked.
|
||||
"""
|
||||
control: Final = _tool_config_point_control(points)
|
||||
tool_list: Final = validated_object_list(tools) or ()
|
||||
tool_fields: Final = tuple(validated_object_mapping(tool) for tool in tool_list)
|
||||
if control is None or any(fields is not None and "cache_control" in fields for fields in tool_fields):
|
||||
return None
|
||||
cacheable: Final = tuple(
|
||||
(position, fields)
|
||||
for position, fields in enumerate(tool_fields)
|
||||
if fields is not None and fields.get("type") not in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
|
||||
)
|
||||
if not cacheable:
|
||||
return None
|
||||
position, fields = cacheable[-1]
|
||||
return (*tool_list[:position], {**fields, "cache_control": control}, *tool_list[position + 1 :])
|
||||
|
||||
|
||||
def is_claude_code_user_agent(user_agent: str) -> bool:
|
||||
"""Claude Code sends its API calls through the Anthropic SDK as `claude-cli/<version>` and its own
|
||||
fetches, such as gateway model discovery, as `claude-code/<version>`"""
|
||||
|
|
@ -173,9 +208,9 @@ def validated_object_mapping(value: object) -> dict[object, object] | None:
|
|||
return None
|
||||
|
||||
|
||||
def _validated_claude_code_list(value: object) -> list[object] | None:
|
||||
def validated_object_list(value: object) -> Sequence[object] | None:
|
||||
try:
|
||||
return _CLAUDE_CODE_OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
return OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
|
@ -202,7 +237,7 @@ def _claude_code_billing_fields(text: str) -> tuple[tuple[str, str], ...] | None
|
|||
def _claude_code_billing_texts(system: object) -> tuple[str, ...] | None:
|
||||
if isinstance(system, str):
|
||||
return (system,)
|
||||
blocks: Final = _validated_claude_code_list(system)
|
||||
blocks: Final = validated_object_list(system)
|
||||
if blocks is None:
|
||||
return None
|
||||
block_mappings: Final = tuple(validated_object_mapping(block) for block in blocks)
|
||||
|
|
@ -1963,8 +1998,8 @@ def tool_call_is_rebuilt_as_server_tool_use(
|
|||
return (
|
||||
find_anthropic_server_tool_result(
|
||||
tool_call_id,
|
||||
_validated_claude_code_list(fields.get("web_search_results")),
|
||||
_validated_claude_code_list(fields.get("tool_results")),
|
||||
validated_object_list(fields.get("web_search_results")),
|
||||
validated_object_list(fields.get("tool_results")),
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
|
|
|||
|
|
@ -14,10 +14,12 @@ from typing import Final, cast
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
strip_provider_specific_fields_from_anthropic_messages,
|
||||
tools_marked_for_a_tool_config_point,
|
||||
)
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
|
|
@ -667,7 +669,13 @@ def anthropic_messages_handler(
|
|||
f"custom_llm_provider is required for Anthropic messages, passed in model={model}, custom_llm_provider={custom_llm_provider}"
|
||||
)
|
||||
|
||||
local_vars.update(kwargs)
|
||||
request_kwargs: Final = cast(dict[str, object], kwargs) # cast-ok: **kwargs of the handler is untyped
|
||||
marked_tools: Final = tools_marked_for_a_tool_config_point(
|
||||
local_vars.get("tools"), request_kwargs.pop("cache_control_injection_points", None)
|
||||
)
|
||||
if marked_tools is not None:
|
||||
AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1)
|
||||
local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})})
|
||||
anthropic_messages_optional_request_params: Final = (
|
||||
AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
|
||||
params=local_vars,
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ class CacheControlMessageInjectionPoint(TypedDict):
|
|||
|
||||
|
||||
class CacheControlToolConfigInjectionPoint(TypedDict):
|
||||
"""Type for tool_config-level injection points (Bedrock)."""
|
||||
"""Type for tool_config-level injection points (Anthropic and Bedrock)."""
|
||||
|
||||
location: Literal["tool_config"]
|
||||
control: ChatCompletionCachedContent | None
|
||||
|
|
|
|||
|
|
@ -5174,6 +5174,9 @@ def get_optional_params(
|
|||
EXTRA_BODY_ROUTING_KEYS: Final = frozenset({"model"})
|
||||
|
||||
|
||||
LITELLM_ONLY_PARAMS: Final = frozenset(("cache_control_injection_points",))
|
||||
|
||||
|
||||
def add_provider_specific_params_to_optional_params(
|
||||
optional_params: dict,
|
||||
passed_params: dict,
|
||||
|
|
@ -5197,7 +5200,7 @@ def add_provider_specific_params_to_optional_params(
|
|||
if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False:
|
||||
extra_body: Final = dict(passed_params.pop("extra_body", None) or {})
|
||||
for k in passed_params:
|
||||
if k not in openai_params and passed_params[k] is not None:
|
||||
if k not in openai_params and k not in LITELLM_ONLY_PARAMS and passed_params[k] is not None:
|
||||
extra_body[k] = passed_params[k]
|
||||
if not isinstance(optional_params.get("extra_body"), dict):
|
||||
optional_params["extra_body"] = {}
|
||||
|
|
|
|||
312
tests/integration/providers/test_tool_config_cache_point_wire.py
Normal file
312
tests/integration/providers/test_tool_config_cache_point_wire.py
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
import json
|
||||
import uuid
|
||||
from typing import Final
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.openai_wire import chat_reply, responses_reply
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_ANTHROPIC_MODEL: Final = "claude-sonnet-4-6"
|
||||
_BEDROCK_MODEL: Final = "us.anthropic.claude-sonnet-4-6"
|
||||
_OPENAI_MODEL: Final = "gpt-4.1"
|
||||
_GEMINI_MODEL: Final = "gemini-3.8-flash"
|
||||
_NOVA_MODEL: Final = "us.amazon.nova-pro-v1:0"
|
||||
_API_KEY: Final = "synthetic-provider-key"
|
||||
_CONTROL: Final = {"type": "ephemeral", "ttl": "1h"}
|
||||
_TOOL_CONFIG_POINT: Final = [{"location": "tool_config", "control": _CONTROL}]
|
||||
_AWS: Final = {
|
||||
"aws_access_key_id": "AKIAINTEGRATIONTEST",
|
||||
"aws_secret_access_key": "integration-secret",
|
||||
"aws_region_name": "us-east-1",
|
||||
}
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_TOOL_LIST: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
|
||||
|
||||
def _openai_tools() -> list[JsonValue]:
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": name,
|
||||
"description": f"{name} tool",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
for name in ("get_weather", "get_forecast")
|
||||
]
|
||||
|
||||
|
||||
def _anthropic_tools() -> list[JsonValue]:
|
||||
return [
|
||||
{"name": name, "description": f"{name} tool", "input_schema": {"type": "object", "properties": {}}}
|
||||
for name in ("get_weather", "get_forecast")
|
||||
]
|
||||
|
||||
|
||||
def _anthropic_reply(identity: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": identity,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": _ANTHROPIC_MODEL,
|
||||
"content": [{"type": "text", "text": "done"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 12, "output_tokens": 3, "cache_creation_input_tokens": 10},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _converse_reply() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "done"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 12, "outputTokens": 3, "totalTokens": 15},
|
||||
"metrics": {"latencyMs": 1},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _assert_last_anthropic_tool_marked(body: dict[str, JsonValue]) -> None:
|
||||
assert "cache_control_injection_points" not in body, body
|
||||
tools: Final = _TOOL_LIST.validate_python(body["tools"])
|
||||
assert [tool["name"] for tool in tools] == ["get_weather", "get_forecast"], tools
|
||||
assert "cache_control" not in tools[0], tools
|
||||
assert tools[1]["cache_control"] == _CONTROL, tools
|
||||
|
||||
|
||||
def test_chat_completions_tool_config_point_marks_the_last_anthropic_tool(gateway: Gateway) -> None:
|
||||
identity: Final = f"anthropic-tool-config-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/v1/messages", request.target
|
||||
_assert_last_anthropic_tool_marked(_JSON_OBJECT.validate_json(request.body))
|
||||
return Reply(body=_anthropic_reply(identity))
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"anthropic/{_ANTHROPIC_MODEL}",
|
||||
api_base=wire.url,
|
||||
api_key=_API_KEY,
|
||||
cache_control_injection_points=_TOOL_CONFIG_POINT,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": _openai_tools(),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["usage"]["cache_creation_input_tokens"] == 10, response.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
|
||||
def test_messages_tool_config_point_marks_the_last_anthropic_tool(gateway: Gateway) -> None:
|
||||
identity: Final = f"anthropic-native-tool-config-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/v1/messages", request.target
|
||||
_assert_last_anthropic_tool_marked(_JSON_OBJECT.validate_json(request.body))
|
||||
return Reply(body=_anthropic_reply(identity))
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"anthropic/{_ANTHROPIC_MODEL}",
|
||||
api_base=wire.url,
|
||||
api_key=_API_KEY,
|
||||
cache_control_injection_points=_TOOL_CONFIG_POINT,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": _anthropic_tools(),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["usage"]["cache_creation_input_tokens"] == 10, response.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
|
||||
def test_chat_completions_tool_config_point_appends_a_bedrock_converse_cache_point(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
path: Final = unquote(urlsplit(request.target).path)
|
||||
assert request.method == "POST" and path == f"/model/{_BEDROCK_MODEL}/converse", request.target
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
assert "cache_control_injection_points" not in body, body
|
||||
tools: Final = _TOOL_LIST.validate_python(_JSON_OBJECT.validate_python(body["toolConfig"])["tools"])
|
||||
assert [tuple(tool) for tool in tools] == [("toolSpec",), ("toolSpec",), ("cachePoint",)], tools
|
||||
assert tools[2] == {"cachePoint": {"type": "default", "ttl": "1h"}}, tools
|
||||
return Reply(body=_converse_reply())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"bedrock/{_BEDROCK_MODEL}",
|
||||
api_base=wire.url,
|
||||
api_key=None,
|
||||
cache_control_injection_points=_TOOL_CONFIG_POINT,
|
||||
**_AWS,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": _openai_tools(),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
|
||||
def test_chat_completions_tool_config_point_never_reaches_openai(gateway: Gateway) -> None:
|
||||
identity: Final = f"openai-tool-config-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/chat/completions", request.target
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
assert "cache_control_injection_points" not in body, body
|
||||
tools: Final = _TOOL_LIST.validate_python(body["tools"])
|
||||
assert [tuple(sorted(tool)) for tool in tools] == [("function", "type"), ("function", "type")], tools
|
||||
return chat_reply(identity, _OPENAI_MODEL, "done", stream=False)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"openai/{_OPENAI_MODEL}",
|
||||
api_base=wire.url,
|
||||
api_key=_API_KEY,
|
||||
cache_control_injection_points=_TOOL_CONFIG_POINT,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": _openai_tools(),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["id"] == identity, response.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
|
||||
def _gemini_reply() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"candidates": [
|
||||
{"content": {"role": "model", "parts": [{"text": "done"}]}, "finishReason": "STOP", "index": 0}
|
||||
],
|
||||
"usageMetadata": {"promptTokenCount": 12, "candidatesTokenCount": 3, "totalTokenCount": 15},
|
||||
"modelVersion": _GEMINI_MODEL,
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def test_messages_tool_config_point_never_reaches_openai_through_the_responses_bridge(gateway: Gateway) -> None:
|
||||
identity: Final = f"openai-bridge-tool-config-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/responses", request.target
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
assert "cache_control_injection_points" not in body, body
|
||||
return responses_reply(identity, _OPENAI_MODEL, "done", stream=False)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"openai/{_OPENAI_MODEL}",
|
||||
api_base=wire.url,
|
||||
api_key=_API_KEY,
|
||||
cache_control_injection_points=_TOOL_CONFIG_POINT,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": _anthropic_tools(),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
|
||||
def test_messages_tool_config_point_never_reaches_gemini_through_the_chat_bridge(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
path: Final = urlsplit(request.target).path
|
||||
assert request.method == "POST" and path == f"/models/{_GEMINI_MODEL}:generateContent", request.target
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
assert "cache_control_injection_points" not in body, body
|
||||
return Reply(body=_gemini_reply())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"gemini/{_GEMINI_MODEL}",
|
||||
api_base=wire.url,
|
||||
api_key=_API_KEY,
|
||||
cache_control_injection_points=_TOOL_CONFIG_POINT,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": _anthropic_tools(),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
|
||||
def test_messages_tool_config_point_appends_a_bedrock_converse_cache_point_through_the_chat_bridge(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
path: Final = unquote(urlsplit(request.target).path)
|
||||
assert request.method == "POST" and path == f"/model/{_NOVA_MODEL}/converse", request.target
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
assert "cache_control_injection_points" not in body, body
|
||||
tools: Final = _TOOL_LIST.validate_python(_JSON_OBJECT.validate_python(body["toolConfig"])["tools"])
|
||||
assert [tuple(tool) for tool in tools] == [("toolSpec",), ("toolSpec",), ("cachePoint",)], tools
|
||||
assert tools[2] == {"cachePoint": {"type": "default"}}, tools
|
||||
return Reply(body=_converse_reply())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"bedrock/{_NOVA_MODEL}",
|
||||
api_base=wire.url,
|
||||
api_key=None,
|
||||
cache_control_injection_points=_TOOL_CONFIG_POINT,
|
||||
**_AWS,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": _anthropic_tools(),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
|
@ -1294,7 +1294,6 @@ def test_anthropic_structured_output_beta_header():
|
|||
print(f"raw_request_headers: {response['raw_request_headers']}")
|
||||
assert "structured-outputs-2025-11-13" in response["raw_request_headers"]["anthropic-beta"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[
|
||||
|
|
@ -1346,6 +1345,7 @@ def test_opus_uses_native_structured_output(model_name):
|
|||
assert optional_params.get("json_mode") is True
|
||||
|
||||
|
||||
|
||||
def test_native_structured_output_uses_bundled_capability_when_remote_map_lags(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
|
@ -3636,7 +3636,9 @@ def _block_ids(message: dict[str, object], block_type: str, id_key: str) -> list
|
|||
return sorted(block[id_key] for block in message["content"] if block["type"] == block_type)
|
||||
|
||||
|
||||
_SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn([], {"type": "text", "text": "I have no tool that books flights."})
|
||||
_SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn(
|
||||
[], {"type": "text", "text": "I have no tool that books flights."}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -9445,7 +9447,9 @@ def test_map_openai_params_runs_tool_search_on_a_custom_named_deployment_whose_m
|
|||
AzureAnthropicConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="my-claude-deployment", drop_params=False
|
||||
)
|
||||
litellm.register_model({"azure_ai/my-claude-deployment": {"supports_tool_search": True, "litellm_provider": "azure_ai"}})
|
||||
litellm.register_model(
|
||||
{"azure_ai/my-claude-deployment": {"supports_tool_search": True, "litellm_provider": "azure_ai"}}
|
||||
)
|
||||
mapped: Final = AzureAnthropicConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="my-claude-deployment", drop_params=False
|
||||
)
|
||||
|
|
@ -9559,3 +9563,81 @@ def test_non_dict_upstream_body_still_transforms_without_refusal_fields() -> Non
|
|||
)
|
||||
provider_specific_fields = result.choices[0].message.provider_specific_fields or {}
|
||||
assert "stop_details" not in provider_specific_fields
|
||||
|
||||
|
||||
def _two_function_tools() -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": f"tool_{i}", "description": f"Tool {i}", "parameters": {"type": "object"}},
|
||||
}
|
||||
for i in range(2)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"control, expected_control",
|
||||
[
|
||||
(None, {"type": "ephemeral"}),
|
||||
({"type": "ephemeral", "ttl": "1h"}, {"type": "ephemeral", "ttl": "1h"}),
|
||||
],
|
||||
)
|
||||
def test_transform_request_tool_config_point_marks_last_tool_and_drops_the_key(control, expected_control):
|
||||
"""LIT-9880: a tool_config injection point used to be spread into the Anthropic body, which
|
||||
Anthropic rejected with 400 Extra inputs are not permitted."""
|
||||
tools = _two_function_tools()
|
||||
tools_before = copy.deepcopy(tools)
|
||||
point = {"location": "tool_config", **({"control": control} if control is not None else {})}
|
||||
litellm_params = {"metadata": {}, "model_info": {"id": "deployment-1"}}
|
||||
|
||||
data = _make_transform_request(
|
||||
optional_params={"tools": tools, "cache_control_injection_points": [point]},
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
assert "cache_control_injection_points" not in data
|
||||
assert data["tools"][-1]["cache_control"] == expected_control
|
||||
assert "cache_control" not in data["tools"][0]
|
||||
assert tools == tools_before
|
||||
assert litellm_params["metadata"]["litellm_gateway_injected_cache"] == "deployment-1"
|
||||
|
||||
|
||||
def test_transform_request_tool_config_point_keeps_the_clients_tool_marker():
|
||||
tools = _two_function_tools()
|
||||
tools[0]["cache_control"] = {"type": "ephemeral"}
|
||||
|
||||
data = _make_transform_request(
|
||||
optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]},
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert "cache_control_injection_points" not in data
|
||||
assert data["tools"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
assert "cache_control" not in data["tools"][1]
|
||||
|
||||
|
||||
def test_transform_request_tool_config_point_skips_tool_search_tools():
|
||||
tool_search_type = next(iter(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES))
|
||||
tools = [*_two_function_tools(), {"type": tool_search_type, "name": "tool_search"}]
|
||||
|
||||
data = _make_transform_request(
|
||||
optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]},
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert data["tools"][1]["cache_control"] == {"type": "ephemeral"}
|
||||
assert "cache_control" not in data["tools"][2]
|
||||
assert "cache_control" not in data["tools"][0]
|
||||
|
||||
|
||||
def test_transform_request_tool_config_point_without_tools_drops_the_key():
|
||||
litellm_params = {"metadata": {}}
|
||||
|
||||
data = _make_transform_request(
|
||||
optional_params={"cache_control_injection_points": [{"location": "tool_config"}]},
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
assert "cache_control_injection_points" not in data
|
||||
assert "tools" not in data
|
||||
assert litellm_params["metadata"] == {}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
|
@ -1483,6 +1484,7 @@ async def test_provider_messages_api_base_env_is_not_shadowed_by_the_chat_defaul
|
|||
|
||||
assert seen_urls == ["https://deepseek.internal.example/anthropic/v1/messages"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthropic():
|
||||
"""Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21."""
|
||||
|
|
@ -1597,7 +1599,9 @@ def _claude_code_auto_mode_request() -> tuple[list[dict[str, object]], list[dict
|
|||
return safeguards, safeguard_results
|
||||
|
||||
|
||||
def _upstream_answering_with(safeguard_results: list[dict[str, object]], captured: dict[str, object]) -> AsyncHTTPHandler:
|
||||
def _upstream_answering_with(
|
||||
safeguard_results: list[dict[str, object]], captured: dict[str, object]
|
||||
) -> AsyncHTTPHandler:
|
||||
def upstream_records_the_request(request: httpx.Request) -> httpx.Response:
|
||||
captured["body"] = json.loads(request.content)
|
||||
captured["anthropic-beta"] = request.headers.get("anthropic-beta")
|
||||
|
|
@ -1623,7 +1627,9 @@ def _upstream_answering_with(safeguard_results: list[dict[str, object]], capture
|
|||
|
||||
|
||||
_CLIENT_BETA_HEADERS: Final = (
|
||||
pytest.param({"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta"),
|
||||
pytest.param(
|
||||
{"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta"
|
||||
),
|
||||
pytest.param({"anthropic-beta": "interleaved-thinking-2025-05-14"}, id="client_omits_beta"),
|
||||
pytest.param({}, id="client_sends_no_beta_header"),
|
||||
)
|
||||
|
|
@ -1748,7 +1754,11 @@ def _tool_use_sse(model: str) -> bytes:
|
|||
"delta": {"type": "input_json_delta", "partial_json": json.dumps(_CURL_TOOL_USE["input"])},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None}, "usage": {"output_tokens": 4}},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "tool_use", "stop_sequence": None},
|
||||
"usage": {"output_tokens": 4},
|
||||
},
|
||||
{"type": "message_stop"},
|
||||
)
|
||||
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode()
|
||||
|
|
@ -1902,26 +1912,20 @@ async def test_anthropic_messages_litellm_router_latency_metadata_tracking():
|
|||
print("Call kwargs:", json.dumps(call_kwargs, indent=2, default=str))
|
||||
|
||||
# Verify that litellm_metadata was passed and contains _latency_per_deployment
|
||||
assert (
|
||||
"litellm_metadata" in call_kwargs
|
||||
), "litellm_metadata should be passed to anthropic_messages"
|
||||
assert "litellm_metadata" in call_kwargs, "litellm_metadata should be passed to anthropic_messages"
|
||||
|
||||
litellm_metadata = call_kwargs["litellm_metadata"]
|
||||
assert litellm_metadata is not None, "litellm_metadata should not be None"
|
||||
assert isinstance(
|
||||
litellm_metadata, dict
|
||||
), "litellm_metadata should be a dictionary"
|
||||
assert isinstance(litellm_metadata, dict), "litellm_metadata should be a dictionary"
|
||||
|
||||
# Verify _latency_per_deployment is present
|
||||
assert (
|
||||
"_latency_per_deployment" in litellm_metadata
|
||||
), "litellm_metadata should contain _latency_per_deployment field"
|
||||
assert "_latency_per_deployment" in litellm_metadata, (
|
||||
"litellm_metadata should contain _latency_per_deployment field"
|
||||
)
|
||||
|
||||
# Verify the structure of _latency_per_deployment
|
||||
latency_per_deployment = litellm_metadata["_latency_per_deployment"]
|
||||
assert isinstance(
|
||||
latency_per_deployment, dict
|
||||
), "_latency_per_deployment should be a dictionary"
|
||||
assert isinstance(latency_per_deployment, dict), "_latency_per_deployment should be a dictionary"
|
||||
|
||||
print(f"✅ Latency per deployment data: {latency_per_deployment}")
|
||||
|
||||
|
|
@ -2079,9 +2083,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough():
|
|||
when using anthropic.messages.acreate with a bedrock model
|
||||
"""
|
||||
# Mock the get_credentials method
|
||||
with unittest.mock.patch.object(
|
||||
BaseAWSLLM, "get_credentials"
|
||||
) as mock_get_credentials:
|
||||
with unittest.mock.patch.object(BaseAWSLLM, "get_credentials") as mock_get_credentials:
|
||||
# Create a proper mock for credentials with the necessary attributes
|
||||
mock_credentials = unittest.mock.MagicMock()
|
||||
mock_credentials.access_key = "mock_access_key"
|
||||
|
|
@ -2092,9 +2094,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough():
|
|||
# We also need to mock the actual AWS request signing to avoid real API calls
|
||||
with unittest.mock.patch("botocore.auth.SigV4Auth.add_auth"):
|
||||
# Set up mock for AsyncHTTPHandler.post to avoid actual API calls
|
||||
with unittest.mock.patch(
|
||||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post"
|
||||
) as mock_post:
|
||||
with unittest.mock.patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post") as mock_post:
|
||||
# Configure mock response
|
||||
mock_response = unittest.mock.MagicMock()
|
||||
mock_response.raise_for_status = unittest.mock.MagicMock()
|
||||
|
|
@ -2136,9 +2136,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough():
|
|||
|
||||
# Assert that our test credentials were passed correctly
|
||||
for param_name, param_value in aws_params.items():
|
||||
assert (
|
||||
call_args[param_name] == param_value
|
||||
), f"Parameter {param_name} was not passed correctly"
|
||||
assert call_args[param_name] == param_value, f"Parameter {param_name} was not passed correctly"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -2166,11 +2164,8 @@ async def test_anthropic_messages_bedrock_dynamic_region():
|
|||
# Patch necessary AWS components
|
||||
with (
|
||||
unittest.mock.patch("botocore.auth.SigV4Auth.add_auth"),
|
||||
unittest.mock.patch.object(
|
||||
BaseAWSLLM, "get_credentials"
|
||||
) as mock_get_credentials,
|
||||
unittest.mock.patch.object(BaseAWSLLM, "get_credentials") as mock_get_credentials,
|
||||
):
|
||||
|
||||
# Setup mock credentials
|
||||
mock_credentials = unittest.mock.MagicMock()
|
||||
mock_credentials.access_key = "test_access_key"
|
||||
|
|
@ -2199,11 +2194,52 @@ async def test_anthropic_messages_bedrock_dynamic_region():
|
|||
|
||||
# Check that the URL contains the correct region
|
||||
url = call_args.kwargs.get("url", "")
|
||||
assert (
|
||||
f"bedrock-runtime.{test_region}.amazonaws.com" in url
|
||||
), f"URL does not contain the correct region. URL: {url}"
|
||||
assert f"bedrock-runtime.{test_region}.amazonaws.com" in url, (
|
||||
f"URL does not contain the correct region. URL: {url}"
|
||||
)
|
||||
|
||||
# Verify get_credentials was called with the correct region
|
||||
mock_get_credentials.assert_called_once()
|
||||
credentials_args = mock_get_credentials.call_args.kwargs
|
||||
assert credentials_args.get("aws_region_name") == test_region
|
||||
|
||||
|
||||
|
||||
|
||||
def test_native_messages_tool_config_point_marks_last_tool_and_drops_the_key(monkeypatch):
|
||||
"""LIT-9880 on /v1/messages: the hook wrote the tool_config point back into kwargs and the
|
||||
native path dropped it silently, so the tools went out without any cache breakpoint."""
|
||||
from litellm.llms.anthropic.pass_through.messages import handler
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_native(**kwargs):
|
||||
captured["params"] = kwargs["anthropic_messages_optional_request_params"]
|
||||
captured["kwargs"] = kwargs["kwargs"]
|
||||
captured["messages"] = kwargs["messages"]
|
||||
return "native-passthrough"
|
||||
|
||||
monkeypatch.setattr(handler.base_llm_http_handler, "anthropic_messages_handler", fake_native)
|
||||
tools = [{"name": f"tool_{i}", "description": f"Tool {i}", "input_schema": {"type": "object"}} for i in range(2)]
|
||||
tools_before = copy.deepcopy(tools)
|
||||
|
||||
result = handler.anthropic_messages_handler(
|
||||
max_tokens=100,
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
model="anthropic/claude-opus-4-6",
|
||||
api_key="sk-test",
|
||||
tools=tools,
|
||||
litellm_metadata={},
|
||||
cache_control_injection_points=[
|
||||
{"location": "message", "role": "user"},
|
||||
{"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}},
|
||||
],
|
||||
)
|
||||
|
||||
assert result == "native-passthrough"
|
||||
assert "cache_control_injection_points" not in captured["params"]
|
||||
assert "cache_control_injection_points" not in captured["kwargs"]
|
||||
assert captured["params"]["tools"][-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
|
||||
assert "cache_control" not in captured["params"]["tools"][0]
|
||||
assert captured["messages"][0]["content"][-1]["cache_control"] == {"type": "ephemeral"}
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1452,8 +1452,6 @@ def test_get_model_info_bedrock_inference_profile_arn_keeps_regional_row(local_m
|
|||
assert bare["input_cost_per_token"] == base["input_cost_per_token"]
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bare_key",
|
||||
[
|
||||
|
|
@ -2289,9 +2287,7 @@ def test_add_known_models_skips_region_scoped_bedrock_pricing_keys(monkeypatch:
|
|||
def test_utils_defines_tests_without_invoking_them_at_module_scope() -> None:
|
||||
tree: Final = ast.parse(Path(__file__).read_text())
|
||||
defined: Final = frozenset(
|
||||
node.name
|
||||
for node in tree.body
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
node.name for node in tree.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
)
|
||||
invoked: Final = tuple(
|
||||
node.value.func.id
|
||||
|
|
@ -3525,9 +3521,7 @@ class TestExtraBodyCannotOverrideModel:
|
|||
}
|
||||
]
|
||||
|
||||
untouched = litellm.get_optional_params(
|
||||
model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools()
|
||||
)
|
||||
untouched = litellm.get_optional_params(model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools())
|
||||
assert untouched["tools"][0]["function"]["custom_marker"] == "LEAK", untouched
|
||||
|
||||
result = litellm.get_optional_params(
|
||||
|
|
@ -4567,7 +4561,9 @@ def test_is_prompt_caching_valid_prompt_stops_counting_once_the_minimum_is_reach
|
|||
assert is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=long_prompt, min_token_count=1024) is True
|
||||
assert sum(counted_messages) < len(long_prompt), sum(counted_messages)
|
||||
|
||||
full_count = litellm.token_counter(model="claude-opus-4-8", messages=long_prompt, use_default_image_token_count=True)
|
||||
full_count = litellm.token_counter(
|
||||
model="claude-opus-4-8", messages=long_prompt, use_default_image_token_count=True
|
||||
)
|
||||
assert (
|
||||
is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=long_prompt, min_token_count=full_count)
|
||||
is True
|
||||
|
|
@ -5034,7 +5030,13 @@ _SUCCESS_RESPONSES_BY_CALL_TYPE: Final = (
|
|||
pytest.param(EmbeddingResponse(model="text-embedding-3-small"), CallTypes.aembedding, id="embedding"),
|
||||
pytest.param(
|
||||
ResponsesAPIResponse(
|
||||
id="resp_abc", created_at=1, output=[], parallel_tool_calls=False, tool_choice="auto", tools=[], model="gpt-5.6"
|
||||
id="resp_abc",
|
||||
created_at=1,
|
||||
output=[],
|
||||
parallel_tool_calls=False,
|
||||
tool_choice="auto",
|
||||
tools=[],
|
||||
model="gpt-5.6",
|
||||
),
|
||||
CallTypes.aresponses,
|
||||
id="responses",
|
||||
|
|
@ -5062,7 +5064,9 @@ async def test_success_deployment_hook_raising_keeps_response_and_runs_later_hoo
|
|||
|
||||
assert result is response
|
||||
assert second_hook.seen_responses == (response,)
|
||||
failure_logs: Final = tuple(r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message)
|
||||
failure_logs: Final = tuple(
|
||||
r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message
|
||||
)
|
||||
assert len(failure_logs) == 1
|
||||
assert "_ChatShapedSuccessDeploymentHook" in failure_logs[0].message
|
||||
assert str(call_type) in failure_logs[0].message
|
||||
|
|
@ -5199,17 +5203,13 @@ async def test_wrapper_async_logs_converted_responses_stream_with_standard_loggi
|
|||
({"metadata": None}, True, False),
|
||||
],
|
||||
)
|
||||
def test_is_litellm_router_call_is_async_aware(
|
||||
kwargs: Mapping[str, object], is_async: bool, expected: bool
|
||||
) -> None:
|
||||
def test_is_litellm_router_call_is_async_aware(kwargs: Mapping[str, object], is_async: bool, expected: bool) -> None:
|
||||
assert _is_litellm_router_call(kwargs, is_async=is_async) is expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"])
|
||||
async def test_router_aresponses_does_not_run_sdk_retries(
|
||||
monkeypatch: pytest.MonkeyPatch, stream: bool
|
||||
) -> None:
|
||||
async def test_router_aresponses_does_not_run_sdk_retries(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
model_list: Final = [
|
||||
|
|
@ -5223,9 +5223,7 @@ async def test_router_aresponses_does_not_run_sdk_retries(
|
|||
},
|
||||
}
|
||||
]
|
||||
router: Final = litellm.Router(
|
||||
model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True
|
||||
)
|
||||
router: Final = litellm.Router(model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True)
|
||||
|
||||
try:
|
||||
with respx.mock(assert_all_called=True) as respx_mock:
|
||||
|
|
@ -5261,9 +5259,7 @@ def test_router_responses_keeps_sdk_retries_for_sync_router_call(
|
|||
},
|
||||
}
|
||||
]
|
||||
router: Final = litellm.Router(
|
||||
model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True
|
||||
)
|
||||
router: Final = litellm.Router(model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True)
|
||||
|
||||
try:
|
||||
with respx.mock(assert_all_called=True) as respx_mock:
|
||||
|
|
@ -7148,7 +7144,9 @@ def test_function_setup_logs_the_search_query_edit_prompt_and_ocr_document_summa
|
|||
|
||||
|
||||
@pytest.mark.parametrize("original_function", ("atext_completion", "text_completion"))
|
||||
def test_function_setup_without_a_prompt_leaves_the_missing_prompt_to_request_validation(original_function: str) -> None:
|
||||
def test_function_setup_without_a_prompt_leaves_the_missing_prompt_to_request_validation(
|
||||
original_function: str,
|
||||
) -> None:
|
||||
assert _logged_request_messages(original_function, model="gpt-4o") is None
|
||||
|
||||
|
||||
|
|
@ -7190,6 +7188,7 @@ def _vcr_outcome_gate(request, vcr):
|
|||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def setup_and_teardown():
|
||||
"""
|
||||
|
|
@ -7203,6 +7202,7 @@ def setup_and_teardown():
|
|||
loop.close()
|
||||
asyncio.set_event_loop(None)
|
||||
|
||||
|
||||
MODEL: Final = "anthropic/claude-haiku-4-5"
|
||||
|
||||
|
||||
|
|
@ -7212,6 +7212,7 @@ def test_validate_tool_choice_none():
|
|||
result = validate_chat_completion_tool_choice(None, model=MODEL)
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_validate_tool_choice_string():
|
||||
"""Test that string values are returned as-is."""
|
||||
|
|
@ -7219,6 +7220,7 @@ def test_validate_tool_choice_string():
|
|||
assert validate_chat_completion_tool_choice("none", model=MODEL) == "none"
|
||||
assert validate_chat_completion_tool_choice("required", model=MODEL) == "required"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_validate_tool_choice_standard_dict():
|
||||
"""Test standard OpenAI format with function."""
|
||||
|
|
@ -7226,6 +7228,7 @@ def test_validate_tool_choice_standard_dict():
|
|||
result = validate_chat_completion_tool_choice(tool_choice, model=MODEL)
|
||||
assert result == tool_choice
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_validate_tool_choice_cursor_format():
|
||||
"""Cursor IDE format {"type": "auto"} is unwrapped to the bare string."""
|
||||
|
|
@ -7233,6 +7236,7 @@ def test_validate_tool_choice_cursor_format():
|
|||
assert validate_chat_completion_tool_choice({"type": "none"}, model=MODEL) == "none"
|
||||
assert validate_chat_completion_tool_choice({"type": "required"}, model=MODEL) == "required"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
"tool_choice",
|
||||
|
|
@ -7254,6 +7258,7 @@ def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice):
|
|||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.model == MODEL
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize("tool_choice", [123, []])
|
||||
def test_validate_tool_choice_invalid_type_is_a_400(tool_choice):
|
||||
|
|
@ -7264,6 +7269,7 @@ def test_validate_tool_choice_invalid_type_is_a_400(tool_choice):
|
|||
validate_chat_completion_tool_choice(tool_choice, model=MODEL)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_validate_tool_choice_without_model_is_still_a_400():
|
||||
"""Callers that predate the model argument keep getting a 400, with an empty model on the error."""
|
||||
|
|
@ -7272,12 +7278,14 @@ def test_validate_tool_choice_without_model_is_still_a_400():
|
|||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.model == ""
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def _vcr_outcome_gate_local_testing(request, vcr):
|
||||
install_live_call_probe(request, vcr)
|
||||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def isolate_litellm_state():
|
||||
"""
|
||||
|
|
@ -7330,6 +7338,7 @@ def isolate_litellm_state():
|
|||
setattr(litellm, attr, original_value)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
_SCALAR_DEFAULTS = {
|
||||
"num_retries": getattr(litellm, "num_retries", None),
|
||||
"num_retries_per_request": getattr(litellm, "num_retries_per_request", None),
|
||||
|
|
@ -7373,6 +7382,7 @@ def setup_and_teardown_local_testing():
|
|||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7398,6 +7408,7 @@ def test_vertex_projects():
|
|||
assert "vertex_ai_project" in optional_params
|
||||
assert "vertex_ai_location" in optional_params
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7413,6 +7424,7 @@ def test_bedrock_embed_v2_regular():
|
|||
print(f"received optional_params: {optional_params}")
|
||||
assert optional_params == {"dimensions": 512}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7431,6 +7443,7 @@ def test_bedrock_embed_v2_with_drop_params():
|
|||
print(f"received optional_params: {optional_params}")
|
||||
assert optional_params == {"dimensions": 512, "embeddingTypes": ["binary"]}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7452,6 +7465,7 @@ def test_openai_non_text_embedding_3_with_allowed_openai_params():
|
|||
print(f"received optional_params: {optional_params}")
|
||||
assert optional_params.get("dimensions") == 1024
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7506,6 +7520,8 @@ def test_openai_non_text_embedding_3_drop_params_per_call():
|
|||
finally:
|
||||
litellm.drop_params = prev_drop_params
|
||||
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7533,12 +7549,14 @@ def test_openai_non_text_embedding_3_drop_params_global():
|
|||
finally:
|
||||
litellm.drop_params = prev_drop_params
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def _vcr_outcome_gate_search_tests(request, vcr):
|
||||
install_live_call_probe(request, vcr)
|
||||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_search_tests")
|
||||
def test_search_tool_name_in_all_litellm_params():
|
||||
"""
|
||||
|
|
@ -7548,6 +7566,7 @@ def test_search_tool_name_in_all_litellm_params():
|
|||
"""
|
||||
assert "search_tool_name" in all_litellm_params
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_search_tests")
|
||||
def test_filter_out_search_tool_name():
|
||||
"""
|
||||
|
|
@ -7574,6 +7593,7 @@ def test_filter_out_search_tool_name():
|
|||
assert filtered["query"] == "latest ai developments"
|
||||
assert filtered["max_results"] == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nested_wrapper_exits_schedule_one_async_success_log(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Chat over the Responses bridge exits two @client wrappers with one logging object. Issue
|
||||
|
|
@ -7726,9 +7746,7 @@ def test_trimming_with_system_message_within_max_tokens():
|
|||
"content": "This is a medium normal message, let's say litellm is awesome.",
|
||||
},
|
||||
]
|
||||
trimmed_messages = trim_messages(
|
||||
messages, max_tokens=30, model="gpt-4-0613"
|
||||
)
|
||||
trimmed_messages = trim_messages(messages, max_tokens=30, model="gpt-4-0613")
|
||||
assert len(trimmed_messages) == 2
|
||||
assert trimmed_messages[0]["content"] == "This is a short system message"
|
||||
|
||||
|
|
@ -7911,9 +7929,7 @@ def test_aget_valid_models():
|
|||
print(valid_models)
|
||||
|
||||
# list of openai supported llms on litellm
|
||||
expected_models = (
|
||||
litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models
|
||||
)
|
||||
expected_models = litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models
|
||||
|
||||
assert set(valid_models) == set(expected_models)
|
||||
|
||||
|
|
@ -8002,9 +8018,7 @@ def test_get_chat_completion_prompt():
|
|||
prompt_variables=None,
|
||||
)
|
||||
|
||||
assert litellm_logging_obj.messages == [
|
||||
{"role": "user", "content": updated_message}
|
||||
]
|
||||
assert litellm_logging_obj.messages == [{"role": "user", "content": updated_message}]
|
||||
|
||||
|
||||
def test_redact_msgs_from_logs():
|
||||
|
|
@ -8076,9 +8090,7 @@ def test_redact_embedding_response():
|
|||
litellm.turn_off_message_logging = True
|
||||
|
||||
# Create a test EmbeddingResponse with usage data
|
||||
original_usage = litellm.Usage(
|
||||
prompt_tokens=10, completion_tokens=0, total_tokens=10
|
||||
)
|
||||
original_usage = litellm.Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10)
|
||||
original_data = [
|
||||
{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3, 0.4, 0.5]},
|
||||
{"object": "embedding", "index": 1, "embedding": [0.6, 0.7, 0.8, 0.9, 1.0]},
|
||||
|
|
@ -8114,9 +8126,7 @@ def test_redact_embedding_response():
|
|||
|
||||
# Assert the redacted response preserves critical metadata
|
||||
assert _redacted_response_obj.usage == original_usage # usage should be preserved
|
||||
assert (
|
||||
_redacted_response_obj.model == "text-embedding-3-small"
|
||||
) # model should be preserved
|
||||
assert _redacted_response_obj.model == "text-embedding-3-small" # model should be preserved
|
||||
assert _redacted_response_obj.object == "list" # object should be preserved
|
||||
|
||||
# Assert sensitive data is cleared
|
||||
|
|
@ -8170,12 +8180,8 @@ def test_redact_msgs_from_logs_with_dynamic_params():
|
|||
)
|
||||
|
||||
# Test Case 1: standard_callback_dynamic_params = False (or not set)
|
||||
standard_callback_dynamic_params = StandardCallbackDynamicParams(
|
||||
turn_off_message_logging=False
|
||||
)
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = (
|
||||
standard_callback_dynamic_params
|
||||
)
|
||||
standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=False)
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params
|
||||
_redacted_response_obj = redact_message_input_output_from_logging(
|
||||
result=response_obj,
|
||||
model_call_details=litellm_logging_obj.model_call_details,
|
||||
|
|
@ -8184,12 +8190,8 @@ def test_redact_msgs_from_logs_with_dynamic_params():
|
|||
assert _redacted_response_obj.choices[0].message.content == test_content
|
||||
|
||||
# Test Case 2: standard_callback_dynamic_params = True
|
||||
standard_callback_dynamic_params = StandardCallbackDynamicParams(
|
||||
turn_off_message_logging=True
|
||||
)
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = (
|
||||
standard_callback_dynamic_params
|
||||
)
|
||||
standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=True)
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params
|
||||
_redacted_response_obj = redact_message_input_output_from_logging(
|
||||
result=response_obj,
|
||||
model_call_details=litellm_logging_obj.model_call_details,
|
||||
|
|
@ -8200,9 +8202,7 @@ def test_redact_msgs_from_logs_with_dynamic_params():
|
|||
# Test Case 3: standard_callback_dynamic_params does not set turn_off_message_logging
|
||||
# since litellm.turn_off_message_logging is True redaction should occur
|
||||
standard_callback_dynamic_params = StandardCallbackDynamicParams()
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = (
|
||||
standard_callback_dynamic_params
|
||||
)
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params
|
||||
_redacted_response_obj = redact_message_input_output_from_logging(
|
||||
result=response_obj,
|
||||
model_call_details=litellm_logging_obj.model_call_details,
|
||||
|
|
@ -8544,10 +8544,7 @@ def test_models_by_provider():
|
|||
continue
|
||||
elif k == "sample_spec":
|
||||
continue
|
||||
elif (
|
||||
v["litellm_provider"] == "sagemaker"
|
||||
or v["litellm_provider"] == "bedrock_converse"
|
||||
):
|
||||
elif v["litellm_provider"] == "sagemaker" or v["litellm_provider"] == "bedrock_converse":
|
||||
continue
|
||||
elif v.get("mode") in ("search", "decisions", "evaluation"):
|
||||
continue
|
||||
|
|
@ -8555,9 +8552,7 @@ def test_models_by_provider():
|
|||
providers.add(v["litellm_provider"])
|
||||
|
||||
for provider in providers:
|
||||
assert provider in models_by_provider.keys() or JSONProviderRegistry.exists(
|
||||
provider
|
||||
)
|
||||
assert provider in models_by_provider.keys() or JSONProviderRegistry.exists(provider)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -8579,16 +8574,11 @@ def restore_end_user_cost_tracking_flags(monkeypatch: pytest.MonkeyPatch) -> Non
|
|||
({"user_api_key_end_user_id": "123"}, True, None),
|
||||
],
|
||||
)
|
||||
def test_get_end_user_id_for_cost_tracking(
|
||||
litellm_params, disable_end_user_cost_tracking, expected_end_user_id
|
||||
):
|
||||
def test_get_end_user_id_for_cost_tracking(litellm_params, disable_end_user_cost_tracking, expected_end_user_id):
|
||||
from litellm.utils import get_end_user_id_for_cost_tracking
|
||||
|
||||
litellm.disable_end_user_cost_tracking = disable_end_user_cost_tracking
|
||||
assert (
|
||||
get_end_user_id_for_cost_tracking(litellm_params=litellm_params)
|
||||
== expected_end_user_id
|
||||
)
|
||||
assert get_end_user_id_for_cost_tracking(litellm_params=litellm_params) == expected_end_user_id
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_end_user_cost_tracking_flags")
|
||||
|
|
@ -8605,13 +8595,9 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only(
|
|||
):
|
||||
from litellm.utils import get_end_user_id_for_cost_tracking
|
||||
|
||||
litellm.enable_end_user_cost_tracking_prometheus_only = (
|
||||
enable_end_user_cost_tracking_prometheus_only
|
||||
)
|
||||
litellm.enable_end_user_cost_tracking_prometheus_only = enable_end_user_cost_tracking_prometheus_only
|
||||
assert (
|
||||
get_end_user_id_for_cost_tracking(
|
||||
litellm_params=litellm_params, service_type="prometheus"
|
||||
)
|
||||
get_end_user_id_for_cost_tracking(litellm_params=litellm_params, service_type="prometheus")
|
||||
== expected_end_user_id
|
||||
)
|
||||
|
||||
|
|
@ -8626,20 +8612,14 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only(
|
|||
),
|
||||
# Test with only litellm_metadata field (new behavior)
|
||||
(
|
||||
{
|
||||
"litellm_metadata": {
|
||||
"user_api_key_end_user_id": "user_from_litellm_metadata"
|
||||
}
|
||||
},
|
||||
{"litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}},
|
||||
"user_from_litellm_metadata",
|
||||
),
|
||||
# Test with both fields - metadata should take precedence for user_api_key fields
|
||||
(
|
||||
{
|
||||
"metadata": {"user_api_key_end_user_id": "user_from_metadata"},
|
||||
"litellm_metadata": {
|
||||
"user_api_key_end_user_id": "user_from_litellm_metadata"
|
||||
},
|
||||
"litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"},
|
||||
},
|
||||
"user_from_metadata",
|
||||
),
|
||||
|
|
@ -8655,9 +8635,7 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only(
|
|||
(
|
||||
{
|
||||
"metadata": {},
|
||||
"litellm_metadata": {
|
||||
"user_api_key_end_user_id": "user_from_litellm_metadata"
|
||||
},
|
||||
"litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"},
|
||||
},
|
||||
"user_from_litellm_metadata",
|
||||
),
|
||||
|
|
@ -8665,9 +8643,7 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only(
|
|||
({}, None),
|
||||
],
|
||||
)
|
||||
def test_get_end_user_id_for_cost_tracking_metadata_handling(
|
||||
litellm_params, expected_end_user_id
|
||||
):
|
||||
def test_get_end_user_id_for_cost_tracking_metadata_handling(litellm_params, expected_end_user_id):
|
||||
"""
|
||||
Test that get_end_user_id_for_cost_tracking correctly handles both metadata and litellm_metadata
|
||||
fields using the get_litellm_metadata_from_kwargs helper function.
|
||||
|
|
@ -8925,9 +8901,7 @@ def test_add_custom_logger_callback_to_specific_event_e2e_failure(monkeypatch):
|
|||
assert len(litellm.success_callback) == curr_len_success_callback
|
||||
assert len(litellm.failure_callback) == curr_len_failure_callback
|
||||
|
||||
assert any(
|
||||
isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback
|
||||
)
|
||||
assert any(isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -9024,20 +8998,14 @@ def test_validate_user_messages_invalid_content_type():
|
|||
[
|
||||
{
|
||||
"name": "default_on_guardrail",
|
||||
"callbacks": [
|
||||
CustomGuardrail(guardrail_name="test_guardrail", default_on=True)
|
||||
],
|
||||
"callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=True)],
|
||||
"kwargs": {"metadata": {"requester_metadata": {"guardrails": []}}},
|
||||
"expected": ["test_guardrail"],
|
||||
},
|
||||
{
|
||||
"name": "request_specific_guardrail",
|
||||
"callbacks": [
|
||||
CustomGuardrail(guardrail_name="test_guardrail", default_on=False)
|
||||
],
|
||||
"kwargs": {
|
||||
"metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}}
|
||||
},
|
||||
"callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)],
|
||||
"kwargs": {"metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}}},
|
||||
"expected": ["test_guardrail"],
|
||||
},
|
||||
{
|
||||
|
|
@ -9046,18 +9014,12 @@ def test_validate_user_messages_invalid_content_type():
|
|||
CustomGuardrail(guardrail_name="default_guardrail", default_on=True),
|
||||
CustomGuardrail(guardrail_name="request_guardrail", default_on=False),
|
||||
],
|
||||
"kwargs": {
|
||||
"metadata": {
|
||||
"requester_metadata": {"guardrails": ["request_guardrail"]}
|
||||
}
|
||||
},
|
||||
"kwargs": {"metadata": {"requester_metadata": {"guardrails": ["request_guardrail"]}}},
|
||||
"expected": ["default_guardrail", "request_guardrail"],
|
||||
},
|
||||
{
|
||||
"name": "empty_metadata",
|
||||
"callbacks": [
|
||||
CustomGuardrail(guardrail_name="test_guardrail", default_on=False)
|
||||
],
|
||||
"callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)],
|
||||
"kwargs": {},
|
||||
"expected": [],
|
||||
},
|
||||
|
|
@ -9836,9 +9798,7 @@ def test_pre_call_rules_see_messages_passed_positionally(monkeypatch: pytest.Mon
|
|||
)
|
||||
|
||||
with pytest.raises(litellm.APIResponseValidationError, match="failed post-call-rule check"):
|
||||
litellm.completion(
|
||||
"gpt-4o-mini", [{"role": "user", "content": "say something forbidden"}], api_key="sk-test"
|
||||
)
|
||||
litellm.completion("gpt-4o-mini", [{"role": "user", "content": "say something forbidden"}], api_key="sk-test")
|
||||
allowed: Final = litellm.completion("gpt-4o-mini", [{"role": "user", "content": "say hi"}], api_key="sk-test")
|
||||
|
||||
assert route.call_count == 1
|
||||
|
|
@ -9856,3 +9816,18 @@ def test_vertex_llama3_model_info_resolves_without_the_meta_prefix():
|
|||
prefixed_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai")
|
||||
assert litellm.get_model_info(model=short_name, custom_llm_provider="vertex_ai") == prefixed_info, model
|
||||
assert prefixed_info["key"] == f"vertex_ai/{model}", model
|
||||
|
||||
|
||||
def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_extra_body():
|
||||
"""LIT-9880: the hook hands a tool_config point on for the provider step, and OpenAI has no
|
||||
step that consumes it, so it must not ride along in extra_body and 400 at OpenAI."""
|
||||
optional_params = litellm.utils.get_optional_params(
|
||||
model="gpt-4.1",
|
||||
custom_llm_provider="openai",
|
||||
tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}],
|
||||
cache_control_injection_points=[{"location": "tool_config", "control": {"type": "ephemeral"}}],
|
||||
extra_body={"kept": True},
|
||||
)
|
||||
|
||||
assert optional_params["extra_body"] == {"kept": True}
|
||||
assert "cache_control_injection_points" not in optional_params
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue